Add Zooniverse proof of concept

zooniverse/zoonitom/.gitignore

@@ -0,0 +1,3 @@
+*.sqlite3
+tmp/
+data/

zooniverse/zoonitom/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'zoonitom.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

zooniverse/zoonitom/requirements.txt

@@ -0,0 +1,2 @@
+tomtoolkit
+panoptes-client

zooniverse/zoonitom/static/tom_common/css/custom.css

@@ -0,0 +1,32 @@
+/*!
+ * Adapted from Bootstrap v4.6.2 (https://getbootstrap.com/)
+ * Set defaults for custom css
+ */
+:root {
+    --primary: #007bff;
+    --secondary: #6c757d;
+    --success: #28a745;
+    --info: #17a2b8;
+    --warning: #ffc107;
+    --danger: #dc3545;
+    --light: #f8f9fa;
+    --dark: #343a40;
+}
+
+body {
+    font-size: 1rem;
+    font-weight: 400;
+    line-height: 1.5;
+    color: #212529;
+    background-color: #fff;
+}
+
+a {
+    color: #007bff;
+    background-color: transparent;
+}
+
+a:hover {
+    color: #0056b3;
+    text-decoration: underline;
+}

zooniverse/zoonitom/templates/zoo/zooniversetarget_detail.html

@@ -0,0 +1,31 @@
+{% extends "tom_common/base.html" %}
+
+{% block content %}
+<h1>Zooniverse Target Detail</h1>
+
+<div>
+    <h2>{{ object.identifier }}</h2>
+    <ul>
+        <li><strong>Survey:</strong> {{ object.survey }}</li>
+        <li><strong>Created:</strong> {{ object.created }}</li>
+        <li><strong>Updated:</strong> {{ object.updated }}</li>
+        <!-- Add more fields as needed -->
+    </ul>
+    <h3>Subjects</h3>
+    <ul>
+        {% for subject in object.zooniversesubject_set.all %}
+        <li>{{ subject.subject_id }} (<a href="{{ subject.talk_url }}">Zooniverse Talk</a>)
+            <ul>
+                {% for k, v in subject.annotation_totals.items %}
+                <li>{{ k }}: {{ v }}</li>
+                {% endfor %}
+            </ul>
+        </li>
+        {% empty %}
+        <li>No subjects found.</li>
+        {% endfor %}
+    </ul>
+</div>
+
+<a href="{% url 'zooniversetarget_list' %}">Back to list</a>
+{% endblock %}

zooniverse/zoonitom/templates/zoo/zooniversetarget_list.html

@@ -0,0 +1,16 @@
+{% extends "tom_common/base.html" %}
+
+{% block content %}
+<h1>Zooniverse Target List</h1>
+<ul>
+    {% for object in object_list %}
+    <li>
+        <a href="{% url 'zooniverse_target_detail' object.pk %}">
+            {{ object }}
+        </a>
+    </li>
+    {% empty %}
+    <li>No targets available.</li>
+    {% endfor %}
+</ul>
+{% endblock %}

zooniverse/zoonitom/zoo/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

zooniverse/zoonitom/zoo/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ZooConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'zoo'

zooniverse/zoonitom/zoo/client.py

@@ -0,0 +1,17 @@
+from django.conf import settings
+
+from panoptes_client import Panoptes, Project, Workflow
+
+if (
+    settings.ZOONIVERSE_CLIENT_ID
+    and settings.ZOONIVERSE_CLIENT_SECRET
+    and not Panoptes.client().logged_in
+):
+    Panoptes.connect(
+        client_id=settings.ZOONIVERSE_CLIENT_ID,
+        client_secret=settings.ZOONIVERSE_CLIENT_SECRET,
+    )
+
+
+project = Project(settings.ZOONIVERSE_PROJECT_ID)
+workflow = Workflow(settings.ZOONIVERSE_WORKFLOW_ID)

zooniverse/zoonitom/zoo/data_import.py

@@ -0,0 +1,155 @@
+import json
+import logging
+
+from csv import DictReader
+
+from dateutil.parser import parse as date_parse
+
+from zoo.client import project, workflow
+from zoo.models import (
+    ZooniverseClassification,
+    ZooniverseSubject,
+    ZooniverseTarget,
+    ZooniverseSurvey,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def generate_subject_export():
+    return project.generate_export("subjects")
+
+
+def generate_classification_export():
+    return project.generate_export("classifications")
+
+
+def get_subject_export():
+    return project.get_export("subjects").csv_dictreader()
+
+
+def get_classification_export():
+    return project.get_export("classifications").csv_dictreader()
+
+
+def import_classifications():
+    """
+    Downloads the latest workflow classifications export and creates new ZooniverseClassification
+    objects based on it.
+    """
+    existing_classifications = ZooniverseClassification.objects.all().values_list(
+        "classification_id", flat=True
+    )
+    existing_subjects = ZooniverseSubject.objects.all().values_list(
+        "subject_id", flat=True
+    )
+    for c in get_classification_export():
+        classification_id = int(c["classification_id"])
+        subject_id = int(c["subject_ids"])
+        user_id = c["user_id"]
+        if len(user_id) == 0:
+            user_id = None
+        else:
+            user_id = int(user_id)
+
+        if classification_id in existing_classifications:
+            continue
+
+        if subject_id not in existing_subjects:
+            logger.warning(
+                f"Skipping classification {classification_id} for unknown subject {subject_id}"
+            )
+            continue
+
+        subject = ZooniverseSubject.objects.get(subject_id=subject_id)
+
+        annotation = json.loads(c["annotations"])
+        timestamp = date_parse(c["created_at"])
+
+        ZooniverseClassification.objects.create(
+            classification_id=classification_id,
+            subject=subject,
+            user_id=user_id,
+            timestamp=timestamp,
+            annotation=annotation,
+        )
+
+
+def import_subjects(
+    target_identifier=None,
+    survey=None,
+    survey_identifier=None,
+    sequence=None,
+    sequence_identifier=None,
+):
+    """
+    Downloads the latest subjects export and creates new ZooniverseSubject objects.
+
+    Options:
+        - target_identifier: The metadata key name which gives the target/object ID.
+          Any subjects which don't have this metadata key will be skipped.
+        - survey: If this and survey_identifier are both provided, filters subjects
+          to just the ones in the specified survey. If survey_identifier is not provided,
+          assumes all subjects are in the specified survey.
+        - survey_identifier: The metadata key name which gives the survey name.
+        - sequence: If this and sequence_identifier are both provided, filters subjects
+          to just the ones in the specified sequence. Has no effect if sequence_identifier
+          is not provided.
+        - sequence_identifier: the metadata key name which gives the sequence name (i.e.
+          the data release number, sector name, or other grouping).
+    """
+    if survey is not None:
+        survey = ZooniverseSurvey.objects.get_or_create(name=survey)[0]
+
+    existing_subjects = ZooniverseSubject.objects.all()
+    if survey is not None:
+        existing_subjects = existing_subjects.filter(target__survey=survey)
+    existing_subjects = existing_subjects.values_list("subject_id", flat=True)
+
+    count = 0
+    for s in get_subject_export():
+        if count > 100:
+            break
+        subject_id = int(s["subject_id"])
+
+        if subject_id in existing_subjects:
+            continue
+
+        locations = json.loads(s["locations"])
+        metadata = json.loads(s["metadata"])
+
+        if survey_identifier is not None:
+            survey_name = metadata.get(survey_identifier, None)
+            if survey_name is None:
+                continue
+            if survey is None:
+                survey = ZooniverseSurvey.objects.get_or_create(name=survey_name)[0]
+            else:
+                if survey.name != survey_name:
+                    continue
+
+        target = None
+        if target_identifier is not None:
+            target_name = metadata.get(target_identifier, None)
+            if target_name is None:
+                continue
+            target = ZooniverseTarget.objects.get_or_create(
+                survey=survey, identifier=target_name
+            )[0]
+
+        sequence_name = None
+        if sequence_identifier is not None:
+            sequence_name = metadata.get(sequence_identifier, None)
+            if sequence_name is None:
+                continue
+            if sequence is not None and sequence != sequence_name:
+                continue
+
+        ZooniverseSubject.objects.create(
+            subject_id=subject_id,
+            metadata=s["metadata"],
+            data_url=locations["0"],
+            target=target,
+            sequence=sequence_name,
+        )
+        count += 1

zooniverse/zoonitom/zoo/migrations/0001_initial.py

@@ -0,0 +1,77 @@
+# Generated by Django 4.2.23 on 2025-06-26 13:40
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ZooniverseClassification',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('classification_id', models.BigIntegerField(unique=True)),
+                ('user_id', models.BigIntegerField()),
+                ('timestamp', models.DateTimeField()),
+                ('annotation', models.JSONField()),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ZooniverseSurvey',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=50)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ZooniverseTarget',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('identifier', models.CharField(max_length=128)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('survey', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversesurvey')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ZooniverseTargetReduction',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('reduced_annotations', models.JSONField()),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('classifications', models.ManyToManyField(to='zoo.zooniverseclassification')),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversetarget')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ZooniverseSubject',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('subject_id', models.BigIntegerField(unique=True)),
+                ('sequence', models.CharField(blank=True, help_text='Sector, data release, etc.', max_length=50, null=True)),
+                ('data_url', models.URLField()),
+                ('start_time', models.DateTimeField(blank=True, help_text='Earliest time in the light curve', null=True)),
+                ('end_time', models.DateTimeField(blank=True, help_text='Latest time in the light curve', null=True)),
+                ('metadata', models.JSONField()),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('target', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversetarget')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='zooniverseclassification',
+            name='subject',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='zoo.zooniversesubject'),
+        ),
+    ]

zooniverse/zoonitom/zoo/migrations/0002_alter_zooniverseclassification_user_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.23 on 2025-06-26 14:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('zoo', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='zooniverseclassification',
+            name='user_id',
+            field=models.BigIntegerField(blank=True, null=True),
+        ),
+    ]