Titan Rebuild

.github/workflows/ci.yml

@@ -144,3 +144,133 @@ jobs:
       - name: Report coverage
         run: coverage report
         working-directory: ./backend
+  unit-test-titan:
+    name: Titan (Go) unit tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v3.1.0
+        with:
+          fetch-depth: 0  # Fetch all history for comparing changes
+      - name: Check for Titan changes
+        id: check_changes
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            # For PRs, check if titan/ directory has changes
+            if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q '^titan/'; then
+              echo "changed=true" >> $GITHUB_OUTPUT
+              echo "Titan package has changes"
+            else
+              echo "changed=false" >> $GITHUB_OUTPUT
+              echo "No changes in Titan package, skipping tests"
+            fi
+          else
+            # For pushes to main or manual triggers, always run
+            echo "changed=true" >> $GITHUB_OUTPUT
+          fi
+      - name: Set up Go
+        if: steps.check_changes.outputs.changed == 'true'
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.18'
+          cache-dependency-path: titan/go.sum
+
+      - name: Install ClamAV
+        if: steps.check_changes.outputs.changed == 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clamav clamav-daemon netcat-openbsd
+
+          # Stop the auto-update service
+          sudo systemctl stop clamav-freshclam || true
+          sudo systemctl disable clamav-freshclam || true
+
+          # Create a minimal test database instead of downloading full databases
+          # This approach is based on ClamAV's own unit test setup
+          echo "Creating minimal ClamAV test database..."
+          sudo mkdir -p /var/lib/clamav
+
+          # Create custom signature files (text-based, no CVD needed)
+          # EICAR test signature in hexadecimal format (full 68-byte EICAR string)
+          echo 'EICAR-Test-Signature:0:*:58354f2150254041505b345c505a58353428505e2937434329377d2445494341522d5354414e444152442d414e544956495255532d544553542d46494c452124482b482a' | sudo tee /var/lib/clamav/test.ndb
+
+          # Create a minimal valid HSB signature file (for ClamAV to recognize database directory)
+          echo '# Minimal test database' | sudo tee /var/lib/clamav/test.hsb
+
+          # Disable freshclam (automatic updates) to avoid interfering with our test database
+          sudo systemctl stop clamav-freshclam.service 2>/dev/null || true
+          sudo systemctl disable clamav-freshclam.service 2>/dev/null || true
+
+          sudo chown -R clamav:clamav /var/lib/clamav
+
+          echo "Test database files created:"
+          ls -lh /var/lib/clamav/
+
+          # Configure clamd for TCP and to NOT require standard databases
+          sudo tee -a /etc/clamav/clamd.conf << EOF
+          TCPSocket 3310
+          TCPAddr 127.0.0.1
+          EOF
+
+          # Override systemd conditions that require daily/main CVD files
+          # Create a drop-in override to remove those conditions
+          sudo mkdir -p /etc/systemd/system/clamav-daemon.service.d
+          sudo tee /etc/systemd/system/clamav-daemon.service.d/override.conf << EOF
+          [Unit]
+          # Remove conditions that require standard CVD databases
+          ConditionPathExistsGlob=
+          EOF
+
+          sudo systemctl daemon-reload
+
+          # Start clamd with our minimal test database
+          echo "Starting clamd with minimal test database..."
+          sudo systemctl start clamav-daemon.service          # Wait for clamd to be ready on TCP (should be fast with minimal database)
+          echo "Waiting for clamd to start with minimal test database..."
+          for i in {1..30}; do
+            if echo "PING" | nc -w 1 127.0.0.1 3310 2>/dev/null | grep -q "PONG"; then
+              echo "✓ clamd is ready and responding on TCP port 3310"
+              exit 0
+            fi
+            if [ $((i % 5)) -eq 0 ]; then
+              echo "Still waiting for clamd... ($i/30 attempts)"
+              echo "=== Recent service logs ==="
+              sudo journalctl -u clamav-daemon.service --no-pager -n 10
+            fi
+            sleep 2
+          done
+
+          # If we got here, clamd didn't start in time - fail the build
+          echo "ERROR: clamd failed to start within 60 seconds"
+          echo "=== Service Status ==="
+          sudo systemctl status clamav-daemon.service --no-pager
+          echo "=== Recent Logs ==="
+          sudo journalctl -u clamav-daemon.service -n 100 --no-pager
+          echo "=== Config File ==="
+          sudo cat /etc/clamav/clamd.conf | grep -E "TCPSocket|TCPAddr|DatabaseDirectory"
+          echo "=== Database Files ==="
+          ls -lh /var/lib/clamav/
+          exit 1
+      - name: Run unit tests
+        if: steps.check_changes.outputs.changed == 'true'
+        run: go test -v -race -short -coverprofile=coverage-unit.out ./pkg/...
+        working-directory: ./titan
+      - name: Run integration tests
+        if: steps.check_changes.outputs.changed == 'true'
+        run: go test -v -race -coverprofile=coverage-integration.out ./pkg/...
+        working-directory: ./titan
+        env:
+          REQUIRE_CLAMD: "true"
+          CLAMD_TCP: "true"
+      - name: Report unit test coverage
+        run: |
+          echo "=== Unit Test Coverage ==="
+          go tool cover -func=coverage-unit.out
+        working-directory: ./titan
+      - name: Report integration test coverage
+        run: |
+          echo "=== Integration Test Coverage (All Tests) ==="
+          go tool cover -func=coverage-integration.out
+        working-directory: ./titan

backend/siarnaq/gcloud/titan.py

@@ -1,23 +1,68 @@
 import datetime
 import io
+import json
 
 import google.cloud.storage as storage
 import structlog
 from django.conf import settings
 from google.auth import impersonated_credentials
 from PIL import Image
 
+from siarnaq.gcloud import saturn
+
 logger = structlog.get_logger(__name__)
 
 
 def request_scan(blob: storage.Blob) -> None:
     """Request that Titan scan a blob for malware."""
-    # Titan responds to google.cloud.storage.object.v1.metadataUpdated events via
-    # Eventarc, so it suffices to set the Titan metadata field.
     logger.info("titan_request", message="Requesting scan on file.", file=blob.name)
+
+    # Set metadata to Unverified before publishing to Pub/Sub
     blob.metadata = {"Titan-Status": "Unverified"}
     blob.patch()
 
+    # Publish scan request to Pub/Sub topic
+    if not settings.GCLOUD_ENABLE_ACTIONS:
+        logger.warn("titan_disabled", message="Titan scan queue is disabled.")
+        return
+
+    publish_client = saturn.get_publish_client()
+    topic = publish_client.topic_path(
+        settings.GCLOUD_PROJECT, settings.GCLOUD_TOPIC_SCAN
+    )
+
+    payload = {
+        "bucket": blob.bucket.name,
+        "name": blob.name,
+    }
+
+    try:
+        future = publish_client.publish(
+            topic=topic,
+            data=json.dumps(payload).encode(),
+            ordering_key=settings.GCLOUD_ORDER_SCAN,
+        )
+        message_id = future.result()
+        logger.info(
+            "titan_enqueued",
+            message="Scan request has been queued.",
+            message_id=message_id,
+            bucket=blob.bucket.name,
+            file=blob.name,
+        )
+    except Exception:
+        logger.error(
+            "titan_publish_error",
+            message="Scan request could not be queued.",
+            exc_info=True,
+            bucket=blob.bucket.name,
+            file=blob.name,
+        )
+        publish_client.resume_publish(
+            topic=topic,
+            ordering_key=settings.GCLOUD_ORDER_SCAN,
+        )
+
 
 def get_object(
     bucket: str, name: str, check_safety: bool, get_raw: bool = False

backend/siarnaq/settings.py

@@ -270,8 +270,10 @@ class Local(Base):
     GCLOUD_BUCKET_EPHEMERAL = "nowhere-ephemeral"
     GCLOUD_TOPIC_COMPILE = "nowhere-siarnaq-compile"
     GCLOUD_TOPIC_EXECUTE = "nowhere-siarnaq-execute"
+    GCLOUD_TOPIC_SCAN = "nowhere-siarnaq-scan"
     GCLOUD_ORDER_COMPILE = "compile-order"
     GCLOUD_ORDER_EXECUTE = "execute-order"
+    GCLOUD_ORDER_SCAN = "scan-order"
     GCLOUD_SCHEDULER_PREFIX = "nothing"
     GCLOUD_BRACKET_QUEUE = "nowhere-siarnaq-bracket"
     GCLOUD_RATING_QUEUE = "nowhere-siarnaq-rating"
@@ -336,8 +338,10 @@ class Staging(Base):
     GCLOUD_BUCKET_EPHEMERAL = "mitbattlecode-staging-ephemeral"
     GCLOUD_TOPIC_COMPILE = "staging-siarnaq-compile"
     GCLOUD_TOPIC_EXECUTE = "staging-siarnaq-execute"
+    GCLOUD_TOPIC_SCAN = "staging-siarnaq-scan"
     GCLOUD_ORDER_COMPILE = "compile-order"
     GCLOUD_ORDER_EXECUTE = "execute-order"
+    GCLOUD_ORDER_SCAN = "scan-order"
     GCLOUD_SCHEDULER_PREFIX = "staging"
     GCLOUD_BRACKET_QUEUE = "staging-siarnaq-bracket"
     GCLOUD_RATING_QUEUE = "staging-siarnaq-rating"
@@ -429,8 +433,10 @@ class Production(Base):
     GCLOUD_BUCKET_EPHEMERAL = "mitbattlecode-production-ephemeral"
     GCLOUD_TOPIC_COMPILE = "production-siarnaq-compile"
     GCLOUD_TOPIC_EXECUTE = "production-siarnaq-execute"
+    GCLOUD_TOPIC_SCAN = "production-siarnaq-scan"
     GCLOUD_ORDER_COMPILE = "compile-order"
     GCLOUD_ORDER_EXECUTE = "execute-order"
+    GCLOUD_ORDER_SCAN = "scan-order"
     GCLOUD_SCHEDULER_PREFIX = "production"
     GCLOUD_BRACKET_QUEUE = "production-siarnaq-bracket"
     GCLOUD_RATING_QUEUE = "production-siarnaq-rating"

deploy/galaxy/main.tf

@@ -167,8 +167,9 @@ module "titan" {
   gcp_zone    = var.gcp_zone
   labels      = merge(var.labels, {component="titan"})
 
-  image         = var.titan_image
-  storage_names = [google_storage_bucket.public.name, google_storage_bucket.secure.name]
+  image                  = var.titan_image
+  storage_names          = [google_storage_bucket.public.name, google_storage_bucket.secure.name]
+  pubsub_topic_scan_name = module.siarnaq.pubsub_topic_scan_name
 }
 
 resource "google_secret_manager_secret" "saturn" {

deploy/siarnaq/main.tf

@@ -1,5 +1,5 @@
 locals {
-  pubsub_topics = toset(["compile", "execute"])
+  pubsub_topics = toset(["compile", "execute", "scan"])
 }
 
 resource "google_service_account" "this" {

deploy/siarnaq/outputs.tf

@@ -13,3 +13,7 @@ output "topic_compile_name" {
 output "topic_execute_name" {
   value = google_pubsub_topic.this["execute"].name
 }
+
+output "pubsub_topic_scan_name" {
+  value = google_pubsub_topic.this["scan"].name
+}

deploy/titan/main.tf

@@ -4,12 +4,6 @@ resource "google_service_account" "this" {
   description  = "Service account for perform Titan actions"
 }
 
-resource "google_project_iam_member" "eventarc" {
-  project = var.gcp_project
-  role    = "roles/eventarc.eventReceiver"
-  member  = "serviceAccount:${google_service_account.this.email}"
-}
-
 resource "google_storage_bucket_iam_member" "this" {
   for_each = toset(var.storage_names)
 
@@ -18,42 +12,28 @@ resource "google_storage_bucket_iam_member" "this" {
   member = "serviceAccount:${google_service_account.this.email}"
 }
 
-data "google_storage_project_service_account" "gcs_account" {
-}
-
-resource "google_project_iam_member" "storage_pubsub" {
-  project = var.gcp_project
-  role    = "roles/pubsub.publisher"
-  member  = "serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"
-}
-
-resource "google_eventarc_trigger" "this" {
-  for_each = toset(var.storage_names)
+resource "google_pubsub_subscription" "scan" {
+  name  = "${var.name}-scan"
+  topic = var.pubsub_topic_scan_name
+  labels = var.labels
 
-  name            = "${var.name}-${each.value}"
-  location        = var.gcp_region
-  service_account = google_service_account.this.email
-  labels          = var.labels
+  push_config {
+    push_endpoint = google_cloud_run_service.this.status[0].url
 
-  matching_criteria {
-    attribute = "type"
-    value     = "google.cloud.storage.object.v1.metadataUpdated"
+    oidc_token {
+      service_account_email = google_service_account.this.email
+    }
   }
 
-  matching_criteria {
-    attribute = "bucket"
-    value     = each.value
-  }
+  ack_deadline_seconds       = 600
+  message_retention_duration = "604800s"  # 7 days
 
-  destination {
-    cloud_run_service {
-      service = google_cloud_run_service.this.name
-      region = var.gcp_region
-    }
+  retry_policy {
+    minimum_backoff = "10s"
+    maximum_backoff = "600s"
   }
 
   depends_on = [
-    google_project_iam_member.eventarc,
     google_cloud_run_service_iam_member.this,
   ]
 }

deploy/titan/variables.tf

@@ -32,3 +32,8 @@ variable "storage_names" {
   description = "Name of Google Cloud Storage buckets to be scanned"
   type        = list(string)
 }
+
+variable "pubsub_topic_scan_name" {
+  description = "Name of the Pub/Sub topic for scan requests"
+  type        = string
+}

titan/Dockerfile

@@ -10,7 +10,7 @@ COPY . .
 RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /titan -ldflags="-s -w" ./cmd/titan/main.go
 
 
-FROM clamav/clamav:0.105.1_base
+FROM clamav/clamav:1.4_base
 
 ENV APP_HOME /app
 WORKDIR $APP_HOME