Fix export memory overflows

config/settings/base.py

@@ -290,6 +290,9 @@
 ADMINS = [("Mitchell Kotler", "mitch@muckrock.com")]
 # https://docs.djangoproject.com/en/dev/ref/settings/#managers
 MANAGERS = ADMINS
+# Chunk size for CSV exports using .iterator() to process large querysets
+# without loading all records into memory at once
+CSV_EXPORT_CHUNK_SIZE = env.int("CSV_EXPORT_CHUNK_SIZE", default=2000)
 
 # LOGGING
 # ------------------------------------------------------------------------------

documentcloud/addons/admin.py

@@ -1,4 +1,5 @@
 # Django
+from django.conf import settings
 from django.contrib import admin, messages
 from django.db.models import JSONField
 from django.forms import widgets
@@ -128,7 +129,10 @@ def export_runs_as_csv(self, request, queryset):
         """Export selected Add-On Runs to CSV."""
         field_names = [
             "addon_id",
+            "addon_name",
             "user_id",
+            "user_name",
+            "user_email",
             "run_id",
             "status",
             "rating",
@@ -143,11 +147,28 @@ def export_runs_as_csv(self, request, queryset):
         writer = csv.writer(response)
         writer.writerow(field_names)
 
-        for run in queryset:
+        limited_queryset = queryset.select_related("addon", "user").only(
+            "addon_id",
+            "user_id",
+            "run_id",
+            "status",
+            "rating",
+            "credits_spent",
+            "created_at",
+            "updated_at",
+            "addon__name",
+            "user__name",
+            "user__email",
+        )
+
+        for run in limited_queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE):
             writer.writerow(
                 [
                     run.addon_id,
+                    run.addon.name,
                     run.user_id,
+                    run.user.name,
+                    run.user.email,
                     run.run_id,
                     run.status,
                     run.rating,

documentcloud/organizations/admin.py

@@ -1,4 +1,5 @@
 # Django
+from django.conf import settings
 from django.contrib import admin
 from django.http import HttpResponse
 
@@ -41,7 +42,7 @@ def export_ai_credit_logs(self, request, queryset):
         writer = csv.writer(response)
         writer.writerow(field_names)
 
-        for log in queryset:
+        for log in queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE):
             writer.writerow(
                 [
                     str(log.organization),

documentcloud/statistics/admin.py

@@ -1,4 +1,5 @@
 # Django
+from django.conf import settings
 from django.contrib import admin
 from django.http import HttpResponse
 
@@ -29,7 +30,7 @@ def export_statistics_as_csv(self, request, queryset):
         writer = csv.writer(response)
         writer.writerow(field_names)
 
-        for obj in queryset:
+        for obj in queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE):
             row = []
             for field_name in field_names:
                 value = getattr(obj, field_name)

documentcloud/users/admin.py

@@ -1,4 +1,5 @@
 # Django
+from django.conf import settings
 from django.contrib import admin
 from django.http.response import HttpResponse
 from django.urls.conf import re_path
@@ -47,7 +48,7 @@ def format_date(date):
         writer.writerow(["username", "name", "email", "last_login", "date_joined"])
         for user in User.objects.only(
             "username", "name", "email", "last_login", "created_at"
-        ).iterator(chunk_size=2000):
+        ).iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE):
             writer.writerow(
                 [
                     user.username,