feat(ci): add staging test workflow and harden staging infrastructure

- Add staging-tests.yaml workflow (trigger: label, /test-staging comment, manual dispatch)
- Concurrency group ensures only one staging test run at a time
- Unified TOKEN passing: all staging make targets accept TOKEN= argument
- Token passed as env var (not CLI arg) to avoid leaking in ps aux
- Fail-fast guard on all targets if TOKEN is missing or empty
- Crash-safe fixture patching: on-disk .staging-backup files self-heal on next run
- .staging-backup files added to .gitignore
- Replaced GD_STAGING_TOKEN with TOKEN env var for consistency across clean/load/test

Author: tychtjan

.github/workflows/staging-tests.yaml

@@ -13,7 +13,7 @@ on:
       test_envs:
         description: 'Tox test environments to run (e.g. py312)'
         required: false
-        default: 'py312'
+        default: 'py314'
       test_filter:
         description: 'Pytest filter expression (-k flag)'
         required: false
@@ -52,20 +52,25 @@ jobs:
       - name: Set up Python
         uses: astral-sh/setup-uv@v6
         with:
-          python-version: '3.12'
+          python-version: '3.14'
 
       - name: Install dependencies
         run: uv sync --all-groups --locked
 
       - name: Clean staging environment
-        run: make clean-staging TOKEN=${{ secrets.PYTHON_SDK_STG_API_KEY }}
+        run: make clean-staging
+        env:
+          TOKEN: ${{ secrets.PYTHON_SDK_STG_API_KEY }}
 
       - name: Load staging environment
-        run: make load-staging TOKEN=${{ secrets.PYTHON_SDK_STG_API_KEY }}
+        run: make load-staging
+        env:
+          TOKEN: ${{ secrets.PYTHON_SDK_STG_API_KEY }}
 
       - name: Run staging tests
         run: |
           make test-staging \
-            TOKEN=${{ secrets.PYTHON_SDK_STG_API_KEY }} \
-            TEST_ENVS=${{ github.event.inputs.test_envs || 'py312' }} \
+            TEST_ENVS=${{ github.event.inputs.test_envs || 'py314' }} \
             ADD_ARGS="${{ github.event.inputs.test_filter && format('-k {0}', github.event.inputs.test_filter) || '' }}"
+        env:
+          TOKEN: ${{ secrets.PYTHON_SDK_STG_API_KEY }}

.gitignore

@@ -34,3 +34,6 @@ docs/.hugo_build.lock
 
 # Export artifacts from Docker export-controller service
 packages/gooddata-sdk/tests/export/exports/default/
+
+# Staging test fixture backups (created by conftest.py, self-heal on next run)
+*.staging-backup

Makefile

@@ -84,14 +84,17 @@ test:
 
 .PHONY: test-staging
 test-staging:
+	@test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required. Usage: make test-staging TOKEN=<api-token>" && exit 1)
 	$(MAKE) -C packages/gooddata-sdk test-staging TOKEN=$(TOKEN)
 
 .PHONY: clean-staging
 clean-staging:
+	@test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required. Usage: make clean-staging TOKEN=<api-token>" && exit 1)
 	cd packages/tests-support && STAGING=1 TOKEN="$(TOKEN)" python clean_staging.py
 
 .PHONY: load-staging
 load-staging:
+	@test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required. Usage: make load-staging TOKEN=<api-token>" && exit 1)
 	cd packages/tests-support && STAGING=1 TOKEN="$(TOKEN)" python upload_demo_layout.py
 
 .PHONY: release

packages/gooddata-sdk/tests/conftest.py

@@ -43,12 +43,6 @@ def pytest_addoption(parser):
         default=str(default_config_path),
         help="Absolut path to test configuration",
     )
-    parser.addoption(
-        "--gd-test-token",
-        action="store",
-        default="",
-        help="API token for staging tests",
-    )
 
 
 @pytest.fixture(scope="session")
@@ -57,10 +51,11 @@ def test_config(request):
     with open(config_path) as f:
         config = yaml.safe_load(f)
 
-    # Override token from CLI argument (staging tests pass it via --gd-test-token)
-    cli_token = request.config.getoption("--gd-test-token")
-    if cli_token:
-        config["token"] = cli_token
+    # Override token from TOKEN env var (set by make test-staging TOKEN=...)
+    if config.get("staging", False):
+        env_token = os.environ.get("TOKEN")
+        if env_token:
+            config["token"] = env_token
 
     return config
 
@@ -179,20 +174,47 @@ def staging_preflight(test_config):
 ]
 
 
-def _patch_file_for_staging(file_path: Path) -> str | None:
-    """Replace local JDBC URL/username with staging values. Returns original content for restore."""
+_STAGING_BACKUP_SUFFIX = ".staging-backup"
+
+
+def _backup_path(file_path: Path) -> Path:
+    return file_path.with_suffix(file_path.suffix + _STAGING_BACKUP_SUFFIX)
+
+
+def _restore_from_backup(file_path: Path) -> None:
+    """Restore a file from its backup (left over from a previous interrupted run)."""
+    backup = _backup_path(file_path)
+    if backup.exists():
+        file_path.write_text(backup.read_text())
+        backup.unlink()
+        logger.info(f"Restored from stale backup: {file_path}")
+
+
+def _patch_file_for_staging(file_path: Path) -> bool:
+    """Replace local JDBC URL/username with staging values. Writes backup to disk for crash safety."""
     if not file_path.exists():
-        return None
+        return False
     original = file_path.read_text()
     patched = original.replace(_LOCAL_DS_URL, _STAGING_DS_URL).replace(
         f"username: {_LOCAL_DS_USERNAME}", f"username: {_STAGING_DS_USERNAME}"
     )
     # Also handle JSON format (username as a JSON field)
     patched = patched.replace(f'"username": "{_LOCAL_DS_USERNAME}"', f'"username": "{_STAGING_DS_USERNAME}"')
     if patched != original:
+        _backup_path(file_path).write_text(original)
         file_path.write_text(patched)
         logger.info(f"Patched for staging: {file_path}")
-    return original
+        return True
+    return False
+
+
+def _restore_patched_file(file_path: Path) -> None:
+    """Restore a file from its backup and remove the backup."""
+    backup = _backup_path(file_path)
+    if backup.exists():
+        file_path.write_text(backup.read_text())
+        backup.unlink()
+        logger.info(f"Restored original: {file_path}")
 
 
 def _find_gooddata_layouts_dirs() -> list[Path]:
@@ -236,21 +258,21 @@ def staging_patch_fixtures(test_config, staging_preflight):
     - Copies gooddata_layouts/default/ -> gooddata_layouts/<org_id>/ so the SDK
       can find layout files (it uses the org ID as the directory name)
 
-    Restores everything on teardown (regardless of test outcome).
+    Uses on-disk backups so that interrupted runs self-heal on the next start.
     """
+    # Always restore leftover backups from a previous interrupted run
+    for fpath in _STAGING_PATCH_FILES:
+        _restore_from_backup(fpath)
+
     if not test_config.get("staging", False):
         yield
         return
 
     import shutil
 
     # 1. Patch JDBC connection strings in fixture files
-    originals: dict[Path, str] = {}
-    for fpath in _STAGING_PATCH_FILES:
-        original = _patch_file_for_staging(fpath)
-        if original is not None:
-            originals[fpath] = original
-    logger.info(f"Patched {len(originals)} fixture files for staging")
+    patched_files = [fpath for fpath in _STAGING_PATCH_FILES if _patch_file_for_staging(fpath)]
+    logger.info(f"Patched {len(patched_files)} fixture files for staging")
 
     # 2. Copy layout dirs (gooddata_layouts/default -> gooddata_layouts/<org_id>)
     from gooddata_sdk import GoodDataSdk
@@ -262,10 +284,9 @@ def staging_patch_fixtures(test_config, staging_preflight):
 
     yield
 
-    # Restore file contents
-    for fpath, content in originals.items():
-        fpath.write_text(content)
-        logger.info(f"Restored original: {fpath}")
+    # Restore patched files from backups
+    for fpath in patched_files:
+        _restore_patched_file(fpath)
 
     # Remove copied directories
     for d in copied_dirs:

packages/gooddata-sdk/tests/gd_test_config_staging.yaml

@@ -1,6 +1,6 @@
 # (C) 2024 GoodData Corporation
 # Staging test configuration.
-# Token is passed via make test-staging TOKEN=... (see conftest.py).
+# Token is passed via TOKEN env var from make test-staging TOKEN=... (see conftest.py).
 staging: true
 host: "https://python-sdk-dex.dev-latest.stg11.panther.intgdc.com"
 token: ""

packages/gooddata-sdk/tox.ini

@@ -10,6 +10,7 @@ dependency_groups =
     test
 pass_env =
     OVERWRITE
+    TOKEN
 setenv =
     COVERAGE_CORE=sysmon
 commands =

project_common.mk

@@ -68,7 +68,8 @@ test-ci:
 
 .PHONY: test-staging
 test-staging:
-	STAGING=1 uv run tox -v $(TOX_FLAGS) $(LOCAL_TEST_ENVS) -- --gd-test-config=tests/gd_test_config_staging.yaml --gd-test-token=$(TOKEN) $(LOCAL_ADD_ARGS)
+	@test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required. Usage: make test-staging TOKEN=<api-token>" && exit 1)
+	TOKEN=$(TOKEN) STAGING=1 uv run tox -v $(TOX_FLAGS) $(LOCAL_TEST_ENVS) -- --gd-test-config=tests/gd_test_config_staging.yaml $(LOCAL_ADD_ARGS)
 
 # this is effective for gooddata-sdk only now - it should be part of test fixtures
 # remove this target once implemented in pytest global fixture