bug/HOP-57

hospexplorer/ask/admin.py

@@ -1,10 +1,14 @@
 import logging
+import threading
 
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.models import User
+from django.db import transaction
+
 from ask.models import Conversation, TermsAcceptance, QARecord, SimWorkflow, WebsiteResource, PDFResource
-from ask.kb_connector import add_website_to_kb, add_pdf_to_kb, delete_kb_document
+from ask.kb_connector import delete_kb_document
+from ask.tasks import run_kb_resource_upload
 
 logger = logging.getLogger(__name__)
 
@@ -179,9 +183,10 @@ def delete_queryset(self, request, queryset):
 
 @admin.register(WebsiteResource)
 class WebsiteResourceAdmin(KBDeleteAdminMixin, admin.ModelAdmin):
-    list_display = ("title", "url", "creator", "modified_at")
+    list_display = ("title", "url", "creator", "status", "modified_at")
+    list_filter = ("status",)
     search_fields = ("title", "url")
-    readonly_fields = ("created_at", "modified_at", "creator", "modifier", "mcp_kb_document_id")
+    readonly_fields = ("created_at", "modified_at", "creator", "modifier", "mcp_kb_document_id", "status", "status_message")
     help_texts = {
         "title": "A short name to identify this website resource.",
         "description": "Optional details about what this website covers.",
@@ -199,26 +204,32 @@ def save_model(self, request, obj, form, change):
         if not change:
             obj.creator = request.user
         obj.modifier = request.user
+        obj.status = WebsiteResource.Status.PROCESSING
+        obj.status_message = "Queued for Knowledge Base upload."
         super().save_model(request, obj, form, change)
 
-        # send the website URL to the MCP KB server
-        # errors are logged but don't block the save
-        # is still saved in the internal DB even if the KB is unreachable
-        try:
-            result = add_website_to_kb(obj.url)
-            obj.mcp_kb_document_id = result.get("doc_id")
-            obj.save(update_fields=["mcp_kb_document_id"])
-            self.message_user(request, f"Website '{obj.title}' sent to Knowledge Base (doc_id={obj.mcp_kb_document_id}).")
-        except Exception as e:
-            logger.exception("Failed to send website to KB: %s", obj.url)
-            self.message_user(request, f"Website saved but failed to send to Knowledge Base: {e}", level="warning")
+        # start MCP KB upload in a background thread AFTER the admin's
+        # transaction commits, so a slow MCP round trip wont time out the save
+        transaction.on_commit(
+            lambda: threading.Thread(
+                target=run_kb_resource_upload,
+                args=("website", obj.pk),
+                daemon=True,
+            ).start()
+        )
+        self.message_user(
+            request,
+            f"Website '{obj.title}' saved. Upload to Knowledge Base is running in the background — "
+            "refresh this page to see the final status.",
+        )
 
 
 @admin.register(PDFResource)
 class PDFResourceAdmin(KBDeleteAdminMixin, admin.ModelAdmin):
-    list_display = ("title", "file", "creator", "modified_at")
+    list_display = ("title", "file", "creator", "status", "modified_at")
+    list_filter = ("status",)
     search_fields = ("title",)
-    readonly_fields = ("created_at", "modified_at", "creator", "modifier", "mcp_kb_document_id")
+    readonly_fields = ("created_at", "modified_at", "creator", "modifier", "mcp_kb_document_id", "status", "status_message")
     help_texts = {
         "title": "A short name to identify this PDF resource.",
         "description": "Optional details about what this PDF covers.",
@@ -236,17 +247,19 @@ def save_model(self, request, obj, form, change):
         if not change:
             obj.creator = request.user
         obj.modifier = request.user
+        obj.status = PDFResource.Status.PROCESSING
+        obj.status_message = "Queued for Knowledge Base upload."
         super().save_model(request, obj, form, change)
 
-        try:
-            obj.file.open("rb")
-            file_bytes = obj.file.read()
-            obj.file.close()
-            result = add_pdf_to_kb(file_bytes, obj.file.name.split("/")[-1], obj.title)
-            obj.mcp_kb_document_id = result.get("doc_id")
-            obj.save(update_fields=["mcp_kb_document_id"])
-            self.message_user(request, f"PDF '{obj.title}' sent to Knowledge Base (doc_id={obj.mcp_kb_document_id}).")
-        except Exception as e:
-            logger.exception("Failed to send PDF to KB: %s", obj.file.name)
-            self.message_user(request, f"PDF saved but failed to send to Knowledge Base: {e}", level="warning")
-
+        transaction.on_commit(
+            lambda: threading.Thread(
+                target=run_kb_resource_upload,
+                args=("pdf", obj.pk),
+                daemon=True,
+            ).start()
+        )
+        self.message_user(
+            request,
+            f"PDF '{obj.title}' saved. Upload to Knowledge Base is running in the background — "
+            "refresh this page to see the final status.",
+        )

hospexplorer/ask/migrations/0012_pdfresource_status_pdfresource_status_message_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 6.0.2 on 2026-04-23 15:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ask', '0011_pdfresource'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pdfresource',
+            name='status',
+            field=models.CharField(choices=[('processing', 'Processing'), ('success', 'Success'), ('error', 'Error'), ('warning', 'Warning')], default='success', max_length=20),
+        ),
+        migrations.AddField(
+            model_name='pdfresource',
+            name='status_message',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AddField(
+            model_name='websiteresource',
+            name='status',
+            field=models.CharField(choices=[('processing', 'Processing'), ('success', 'Success'), ('error', 'Error'), ('warning', 'Warning')], default='success', max_length=20),
+        ),
+        migrations.AddField(
+            model_name='websiteresource',
+            name='status_message',
+            field=models.TextField(blank=True, default=''),
+        ),
+    ]

hospexplorer/ask/models.py

@@ -5,6 +5,12 @@
 
 # Abstract Model, fields are inherited by subclasses
 class Resource(models.Model):
+    class Status(models.TextChoices):
+        PROCESSING = "processing", "Processing"
+        SUCCESS = "success", "Success"
+        ERROR = "error", "Error"
+        WARNING = "warning", "Warning"
+
     title = models.CharField(max_length=255)
     description = models.TextField(blank=True, default="")
     creator = models.ForeignKey(
@@ -21,6 +27,12 @@ class Resource(models.Model):
     )
     created_at = models.DateTimeField(auto_now_add=True)
     modified_at = models.DateTimeField(auto_now=True)
+    status = models.CharField(
+        max_length=20,
+        choices=Status.choices,
+        default=Status.SUCCESS,
+    )
+    status_message = models.TextField(blank=True, default="")
 
     class Meta:
         abstract = True

hospexplorer/ask/tasks.py

@@ -1,6 +1,8 @@
 import json
 import logging
 
+import httpx
+from django.conf import settings
 from django.db import close_old_connections
 from django.utils import timezone
 
@@ -140,3 +142,60 @@ def run_llm_task(task_id, record_id, conversation_id):
             logger.exception("Failed to mark task as failed, task_id=%s", task_id)
     finally:
         close_old_connections()
+
+
+def run_kb_resource_upload(model_label, resource_id):
+    """Background thread: push a resource to the MCP KB and record its doc_id.
+
+    Runs outside the admin's atomic save transaction so a slow or timing-out
+    MCP call can't roll back the local row. The object's status/status_message
+    are updated at each phase so the admin can surface progress and errors.
+    """
+    from ask.models import WebsiteResource, PDFResource, Resource
+    from ask.kb_connector import add_pdf_to_kb, add_website_to_kb
+
+    if model_label == "pdf":
+        Model = PDFResource
+    elif model_label == "website":
+        Model = WebsiteResource
+    else:
+        logger.error("run_kb_resource_upload: unknown model_label=%r", model_label)
+        return
+
+    try:
+        obj = Model.objects.get(pk=resource_id)
+    except Model.DoesNotExist:
+        logger.error("run_kb_resource_upload: %s id=%s not found", model_label, resource_id)
+        return
+
+    try:
+        if model_label == "pdf":
+            obj.file.open("rb")
+            try:
+                file_bytes = obj.file.read()
+            finally:
+                obj.file.close()
+            result = add_pdf_to_kb(file_bytes, obj.file.name.split("/")[-1], obj.title)
+        else:
+            result = add_website_to_kb(obj.url)
+
+        obj.mcp_kb_document_id = result.get("doc_id")
+        obj.status = Resource.Status.SUCCESS
+        obj.status_message = f"Uploaded to Knowledge Base (doc_id={obj.mcp_kb_document_id})."
+        obj.save(update_fields=["mcp_kb_document_id", "status", "status_message"])
+    except httpx.TimeoutException:
+        logger.exception("Background KB %s upload timed out for resource_id=%s", model_label, resource_id)
+        obj.status = Resource.Status.ERROR
+        obj.status_message = (
+            f"Upload timed out after {settings.KB_MCP_TIMEOUT}s. "
+            "The Knowledge Base did not finish processing this file in time — "
+            "it may be too large. Edit the resource and save again to retry."
+        )
+        obj.save(update_fields=["status", "status_message"])
+    except Exception as e:
+        logger.exception("Background KB %s upload failed for resource_id=%s", model_label, resource_id)
+        obj.status = Resource.Status.ERROR
+        obj.status_message = f"Upload to Knowledge Base failed: {e}"[:1000]
+        obj.save(update_fields=["status", "status_message"])
+    finally:
+        close_old_connections()