Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/main/python/crowd_server/amt/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ def get_assignment_context(request):
def get_response_context(request):
# Extract data from the request
return {'answers': request.POST.get('answers'),
'metrics': request.POST.get('metrics'),
'task_id': request.POST.get('HITId'),
'worker_id': request.POST.get('workerId'),
'assignment_id': request.POST.get('assignmentId')
Expand Down
3 changes: 2 additions & 1 deletion src/main/python/crowd_server/basecrowd/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ def get_response_context(request):
return {'task_id': request.POST.get('task_id', None),
'worker_id': request.POST.get('worker_id', None),
'assignment_id': request.POST.get('assignment_id', None),
'answers': request.POST.get('answers', None)
'answers': request.POST.get('answers', None),
'metrics': request.POST.get('metrics', None)
}


Expand Down
3 changes: 3 additions & 0 deletions src/main/python/crowd_server/basecrowd/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ class AbstractCrowdWorkerResponse(models.Model):
# The content of the response (specific to the task type).
content = models.TextField()

# The content of recorded experimental metrics
metrics = models.TextField()
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, I wonder if we should normalize metrics as their own model, since we might want to query over them (e.g. 'get the average response time per task grouped by task type').

What do you think? Is there a reason you chose to keep it as denormalized JSON here?


# The assignment id of this response
assignment_id = models.CharField(max_length=200)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@
return $(".label_container :input");
}
{% endblock get_answer_inputs_func %}

{% block get_task_divs_func %}
function get_task_divs()
{
return $(".panel.panel-default");
}
{% endblock get_task_divs_func%}

{% block validate_func %}
function validate_answers()
{
Expand All @@ -32,6 +40,14 @@
return answers;
}
{% endblock build_answer_data_func %}

//make metric dictionary
{% block build_metric_data_func %}
function build_metric_data()
{
return {"time" : time};
}
{% endblock build_metric_data_func %}

{% block get_submit_context_func %}
function get_submit_context(urlParamStrings)
Expand All @@ -47,12 +63,15 @@
{
// build the user's answers into the right format
var answers = build_answer_data();

var metrics = build_metric_data();

// get additional context from the URL or page
var paramstr = window.location.search.substring(1);
var parampairs = paramstr.split("&");
var data = get_submit_context(parampairs);
data.answers = JSON.stringify(answers);
data.metrics = JSON.stringify(metrics);

return data;
}
{% endblock prepare_submit_data_func %}
Expand Down Expand Up @@ -86,6 +105,41 @@
}
{% endblock handle_is_accepted_func %}

{% block initialize_metric_func %}
function initialize_metric()
{
{% block create_global_variables %}

// A map that maps task_id to the time the worker spent on it.
time = {}
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Theoretically we could store any metric in here, right? No reason to limit the metrics to task_ids and timings... a map from 'metric_name' to 'metric_value' is all we need.

// The task_id of the task the mouse is currently on, empty if the mouse is outside any task.
focus = "";
// The update freqency of the setInterval function, in ms.
delay = 10;

{% endblock %}

{% block Register_event_listeners %}

// Register a mouseenter event listener for each task divs.
get_task_divs().mouseenter(function(){
focus = $(this).find(":radio:first").attr("id");
});

// Use setInterval to track timing
setInterval(function(){
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This feels like an odd implementation choice. You're already using mouseenter event listeners to track focus, why not just track timing inside the event functions? For example, on mouseenter, record the focus and the timestamp at which the focus started, and on mouseleave, take a new timestamp and determine how long the focus was held.

In general, setInterval is a pretty ugly solution, as it constantly churns the CPU even when the user isn't doing anything.

if (focus != "")
if (focus in time)
time[focus] += delay;
else
time[focus] = delay;
}, delay);

{% endblock %}
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We also might as well record total time on the page here, right? I don't think it's necessarily the sum of all the values in the time dictionary, so we should explicitly track it.


}
{% endblock initialize_metric_func %}

{% block initialize_form_validation_func %}
function initialize_form_validation()
{
Expand Down Expand Up @@ -137,6 +191,9 @@

// Prepare form validation
initialize_form_validation();

// Prepare metric collection
initialize_metric();
});
{% endblock ready_func %}
</script>
Expand Down Expand Up @@ -179,15 +236,15 @@
<div class="panel-body">
{% block instruction %}

{% if group_context.instruction %}
{{group_context.instruction | safe}}
{% else %}
{% block default_instruction %}
Hey, I am an instruction!
{% if group_context.instruction %}
{{group_context.instruction | safe}}
{% else %}
{% block default_instruction %}
Hey, I am an instruction!
{% endblock %}
{% endif %}

{% endblock %}
{% endblock %}
</div>
</div>
{% endblock instruction_panel %}
Expand Down
3 changes: 2 additions & 1 deletion src/main/python/crowd_server/basecrowd/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ def post_response(request, crowd_name):

# validate context
interface.require_context(
context, ['assignment_id', 'task_id', 'worker_id', 'answers'],
context, ['assignment_id', 'task_id', 'worker_id', 'answers', 'metrics'],
ValueError("Response context missing required keys."))

# Check if this is a duplicate response
Expand All @@ -196,6 +196,7 @@ def post_response(request, crowd_name):
task=current_task,
worker=current_worker,
content=context['answers'],
metrics=context['metrics'],
assignment_id=context['assignment_id'])
interface.response_pre_save(current_response)
current_response.save()
Expand Down