chore(tests): three-way Better Stack split — SDK / Platform API / applications

ari-nz · ari-nz · commit 129c9aa7b28a · 2026-05-12T10:45:35.000+02:00
Tests are now routed to three separate Better Stack heartbeat monitors:

- SDK (no monitors marker): token management, service wiring
- Platform API (monitors_platform_api): auth, listing, connectivity
  - test_cli_health_json (system)
  - test_cli_application_list_verbose (application)
  - test_cli_run_list_limit_10 (application)
- HE-TME / applications (monitors): application processing
  - existing he-tme and test-app tests (unchanged)

Adds monitors_platform_api pytest marker alongside the existing
monitors("platform-api") string for pytest -m expression filtering.
Adds BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_* secrets (optional,
heartbeat step skips gracefully when absent).
diff --git a/.github/workflows/_scheduled-test-hourly.yml b/.github/workflows/_scheduled-test-hourly.yml
@@ -25,6 +25,9 @@ on:
       BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING:
         # Optional until the HE-TME Better Stack monitor is created; the heartbeat step skips gracefully when absent.
         required: false
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING:
+        # Optional until the Platform API Better Stack monitor is created; the heartbeat step skips gracefully when absent.
+        required: false
       AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION:
         required: true
       AIGNOSTICS_REFRESH_TOKEN_PRODUCTION:
@@ -36,6 +39,9 @@ on:
       BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION:
         # Optional until the HE-TME Better Stack monitor is created; the heartbeat step skips gracefully when absent.
         required: false
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION:
+        # Optional until the Platform API Better Stack monitor is created; the heartbeat step skips gracefully when absent.
+        required: false
       SENTRY_DSN:
         required: true
 
@@ -96,10 +102,12 @@ jobs:
           echo "$GCP_CREDENTIALS" | base64 -d > credentials.json
           echo "GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/credentials.json" >> $GITHUB_ENV
 
-      # Tests are split into two runs so failures can be routed to the correct Better Stack monitor:
-      # SDK-layer failures (auth, listing, connectivity) go to the SDK monitor; HE-TME application
-      # failures go to the HE-TME monitor. Tests declare which system they monitor via
-      # @pytest.mark.monitors("he-tme"); tests without that marker are SDK health checks.
+      # Tests are split into three runs so failures can be routed to the correct Better Stack monitor:
+      #   - SDK-layer failures (token management, service wiring): no monitors marker → SDK monitor
+      #   - Platform API failures (auth, listing, connectivity): monitors_platform_api → Platform API monitor
+      #   - Application failures (HE-TME, test-app processing): monitors("he-tme"/"test-app") → HE-TME monitor
+      # Tests declare which system they monitor via @pytest.mark.monitors("he-tme") /
+      # @pytest.mark.monitors_platform_api; tests with neither marker are SDK health checks.
       - name: Test / scheduled / sdk
         id: test_sdk
         env:
@@ -110,10 +118,24 @@ jobs:
           # manually and send it to Better Stack regardless of outcome.
           set +e
           XDIST_WORKER_FACTOR=1 uv run --all-extras nox -s test -- \
-            -m "(scheduled or scheduled_only) and not monitors and not stress_only" \
+            -m "(scheduled or scheduled_only) and not monitors and not monitors_platform_api and not stress_only" \
             --junit-xml=reports/junit_sdk.xml
           echo "exit_code=$?" >> $GITHUB_OUTPUT
 
+      - name: Test / scheduled / platform-api
+        id: test_platform_api
+        env:
+          SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+        shell: bash
+        run: |
+          # set +e so a test failure does not abort the step — we capture the exit code
+          # manually and send it to Better Stack regardless of outcome.
+          set +e
+          XDIST_WORKER_FACTOR=1 uv run --all-extras nox -s test -- \
+            -m "(scheduled or scheduled_only) and monitors_platform_api and not stress_only" \
+            --junit-xml=reports/junit_platform_api.xml
+          echo "exit_code=$?" >> $GITHUB_OUTPUT
+
       - name: Test / scheduled / he-tme
         id: test_he_tme
         env:
@@ -124,7 +146,7 @@ jobs:
           # manually and send it to Better Stack regardless of outcome.
           set +e
           XDIST_WORKER_FACTOR=1 uv run --all-extras nox -s test -- \
-            -m "(scheduled or scheduled_only) and monitors and not stress_only" \
+            -m "(scheduled or scheduled_only) and monitors and not monitors_platform_api and not stress_only" \
             --junit-xml=reports/junit_he_tme.xml
           echo "exit_code=$?" >> $GITHUB_OUTPUT
 
@@ -135,14 +157,16 @@ jobs:
           # Default to 1 (failure) if a step never wrote its output — e.g. a setup step
           # errored before tests ran. Prevents sending a false "healthy" heartbeat.
           SDK_EXIT=${{ steps.test_sdk.outputs.exit_code || '1' }}
+          PLATFORM_API_EXIT=${{ steps.test_platform_api.outputs.exit_code || '1' }}
           HE_TME_EXIT=${{ steps.test_he_tme.outputs.exit_code || '1' }}
-          # Combined exit code: non-zero if either run failed
-          if [ "$SDK_EXIT" != "0" ] || [ "$HE_TME_EXIT" != "0" ]; then
+          # Combined exit code: non-zero if any run failed
+          if [ "$SDK_EXIT" != "0" ] || [ "$PLATFORM_API_EXIT" != "0" ] || [ "$HE_TME_EXIT" != "0" ]; then
             COMBINED_EXIT=1
           else
             COMBINED_EXIT=0
           fi
           echo "sdk_exit=${SDK_EXIT}" >> $GITHUB_OUTPUT
+          echo "platform_api_exit=${PLATFORM_API_EXIT}" >> $GITHUB_OUTPUT
           echo "he_tme_exit=${HE_TME_EXIT}" >> $GITHUB_OUTPUT
           echo "combined_exit=${COMBINED_EXIT}" >> $GITHUB_OUTPUT
 
@@ -229,6 +253,56 @@ jobs:
             "${BETTERSTACK_HEARTBEAT_URL}/${SDK_EXIT}"
           echo "INFO: Sent SDK heartbeat to BetterStack with exit code '${SDK_EXIT}'"
 
+      - name: Heartbeat / BetterStack / Platform API
+        if: always()
+        shell: bash
+        env:
+          BETTERSTACK_HEARTBEAT_URL_PLATFORM_API: "${{ inputs.platform_environment == 'staging' && secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING || secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION }}"
+          PLATFORM_API_EXIT: ${{ steps.collect.outputs.platform_api_exit }}
+        run: |
+          if [ -z "$BETTERSTACK_HEARTBEAT_URL_PLATFORM_API" ]; then
+            echo "INFO: No BetterStack Platform API heartbeat URL configured, skipped."
+            exit 0
+          fi
+          BETTERSTACK_METADATA_PAYLOAD=$(jq -n \
+            --arg github_workflow "${{ github.workflow }}" \
+            --arg github_run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
+            --arg github_run_id "${{ github.run_id }}" \
+            --arg github_job "${{ github.job }}" \
+            --arg github_sha "${{ github.sha }}" \
+            --arg github_actor "${{ github.actor }}" \
+            --arg github_repository "${{ github.repository }}" \
+            --arg github_ref "${{ github.ref }}" \
+            --arg job_status "${{ job.status }}" \
+            --arg github_event_name "${{ github.event_name }}" \
+            --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
+            '{
+              github: {
+                workflow: $github_workflow,
+                run_url: $github_run_url,
+                run_id: $github_run_id,
+                job: $github_job,
+                sha: $github_sha,
+                actor: $github_actor,
+                repository: $github_repository,
+                ref: $github_ref,
+                event_name: $github_event_name
+              },
+              job: {
+                status: $job_status,
+              },
+              timestamp: $timestamp,
+            }'
+          )
+          curl \
+            --fail-with-body \
+            --silent \
+            --request POST \
+            --header "Content-Type: application/json" \
+            --data-binary "${BETTERSTACK_METADATA_PAYLOAD}" \
+            "${BETTERSTACK_HEARTBEAT_URL_PLATFORM_API}/${PLATFORM_API_EXIT}"
+          echo "INFO: Sent Platform API heartbeat to BetterStack with exit code '${PLATFORM_API_EXIT}'"
+
       - name: Heartbeat / BetterStack / HE-TME
         if: always()
         shell: bash
@@ -286,6 +360,7 @@ jobs:
           name: test-results-scheduled
           path: |
             reports/junit_sdk.xml
+            reports/junit_platform_api.xml
             reports/junit_he_tme.xml
             reports/coverage.xml
             reports/coverage.md
diff --git a/.github/workflows/scheduled-testing-production-hourly.yml b/.github/workflows/scheduled-testing-production-hourly.yml
@@ -25,9 +25,11 @@ jobs:
       GCP_CREDENTIALS_STAGING: ${{ secrets.GCP_CREDENTIALS_STAGING }}
       BETTERSTACK_HEARTBEAT_URL_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_STAGING }}
       BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING }}
       AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION: ${{ secrets.AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION }}
       AIGNOSTICS_REFRESH_TOKEN_PRODUCTION: ${{ secrets.AIGNOSTICS_REFRESH_TOKEN_PRODUCTION }}
       GCP_CREDENTIALS_PRODUCTION: ${{ secrets.GCP_CREDENTIALS_PRODUCTION }}
       BETTERSTACK_HEARTBEAT_URL_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PRODUCTION }}
       BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }} # For metrics and heartbeat
diff --git a/.github/workflows/scheduled-testing-staging-hourly.yml b/.github/workflows/scheduled-testing-staging-hourly.yml
@@ -25,9 +25,11 @@ jobs:
       GCP_CREDENTIALS_STAGING: ${{ secrets.GCP_CREDENTIALS_STAGING }}
       BETTERSTACK_HEARTBEAT_URL_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_STAGING }}
       BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_STAGING }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_STAGING }}
       AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION: ${{ secrets.AIGNOSTICS_CLIENT_ID_DEVICE_PRODUCTION }}
       AIGNOSTICS_REFRESH_TOKEN_PRODUCTION: ${{ secrets.AIGNOSTICS_REFRESH_TOKEN_PRODUCTION }}
       GCP_CREDENTIALS_PRODUCTION: ${{ secrets.GCP_CREDENTIALS_PRODUCTION }}
       BETTERSTACK_HEARTBEAT_URL_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PRODUCTION }}
       BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_HE_TME_PRODUCTION }}
+      BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_PLATFORM_API_PRODUCTION }}
       SENTRY_DSN: ${{ secrets.SENTRY_DSN }} # For metrics and heartbeat
diff --git a/pyproject.toml b/pyproject.toml
@@ -384,6 +384,7 @@ markers = [
     "integration: Sociable integration tests - test interactions across architectural layers (e.g. CLI/GUI→Service, Service→Utils) or between modules (e.g. Application→Platform), using real SDK collaborators, real file I/O, real subprocesses, and real Docker containers. Integration test must be able to pass offline, i.e. mock external services (Aignostics Platform API, Auth0, S3/GCS buckets, IDC). The timeout should not be bigger than the default 10s, and must be <5 min.",
     "e2e: End-to-end tests - test complete workflows with real external network services (Aignostics Platform API, cloud storage, IDC, etc). If the test timeout is >= 5 min and < 60 min, additionally mark as `long_running`, if >= 60min mark as 'very_long_running'.",
     "monitors: Tag a scheduled test with the application it monitors, e.g. @pytest.mark.monitors('he-tme'). Tests without this marker are considered SDK-layer health checks. Used to route Better Stack heartbeats to the correct monitor.",
+    "monitors_platform_api: Tag a scheduled test that monitors the Platform API layer (auth, listing, connectivity). Used to route Platform API Better Stack heartbeats separately from SDK and application monitors.",
 ]
 md_report = true
 md_report_output = "reports/pytest.md"
diff --git a/tests/aignostics/application/cli_test.py b/tests/aignostics/application/cli_test.py
@@ -164,6 +164,8 @@ def test_cli_application_list_non_verbose(runner: CliRunner, record_property) ->
 
 @pytest.mark.e2e
 @pytest.mark.scheduled
+@pytest.mark.monitors("platform-api")
+@pytest.mark.monitors_platform_api
 @pytest.mark.timeout(timeout=60)
 def test_cli_application_list_verbose(runner: CliRunner, record_property) -> None:
     """Check application list command runs successfully."""
@@ -795,6 +797,8 @@ def test_cli_run_submit_and_describe_and_cancel_and_download_and_delete(  # noqa
 
 @pytest.mark.e2e
 @pytest.mark.scheduled
+@pytest.mark.monitors("platform-api")
+@pytest.mark.monitors_platform_api
 @pytest.mark.timeout(timeout=60)
 def test_cli_run_list_limit_10(runner: CliRunner, record_property) -> None:
     """Check run list command runs successfully."""
diff --git a/tests/aignostics/system/cli_test.py b/tests/aignostics/system/cli_test.py
@@ -43,6 +43,8 @@ def test_cli_health_yaml_format(mock_service: MagicMock, runner: CliRunner, reco
 
 @pytest.mark.e2e
 @pytest.mark.scheduled
+@pytest.mark.monitors("platform-api")
+@pytest.mark.monitors_platform_api
 @pytest.mark.timeout(timeout=60)
 def test_cli_health_json(runner: CliRunner) -> None:
     """Check health CLI returns valid JSON with a valid status value."""

Original file line number	Diff line number	Diff line change
`@@ -384,6 +384,7 @@ markers = [`
`384`	`384`	`"integration: Sociable integration tests - test interactions across architectural layers (e.g. CLI/GUI→Service, Service→Utils) or between modules (e.g. Application→Platform), using real SDK collaborators, real file I/O, real subprocesses, and real Docker containers. Integration test must be able to pass offline, i.e. mock external services (Aignostics Platform API, Auth0, S3/GCS buckets, IDC). The timeout should not be bigger than the default 10s, and must be <5 min.",`
`385`	`385`	"e2e: End-to-end tests - test complete workflows with real external network services (Aignostics Platform API, cloud storage, IDC, etc). If the test timeout is >= 5 min and < 60 min, additionally mark as `long_running`, if >= 60min mark as 'very_long_running'.",
`386`	`386`	`"monitors: Tag a scheduled test with the application it monitors, e.g. @pytest.mark.monitors('he-tme'). Tests without this marker are considered SDK-layer health checks. Used to route Better Stack heartbeats to the correct monitor.",`
	`387`	`+ "monitors_platform_api: Tag a scheduled test that monitors the Platform API layer (auth, listing, connectivity). Used to route Platform API Better Stack heartbeats separately from SDK and application monitors.",`
`387`	`388`	`]`
`388`	`389`	`md_report = true`
`389`	`390`	`md_report_output = "reports/pytest.md"`