Add PPI Validation to UI Workflow

.github/workflows/ci.yml

@@ -39,7 +39,6 @@ jobs:
           key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
             ${{ runner.os }}-poetry-
-
       - name: Install deps
         run: |
           poetry install --with dev

cicd/cloudbuild_dev_and_sandbox.yaml

@@ -0,0 +1,41 @@
+steps:
+  - name: gcr.io/$PROJECT_ID/testrunner
+    args:
+      - '-c'
+      - |
+        set -e
+        echo === Check Tooling Versions ===
+        gcloud --version
+        make --version
+        pip --version
+        python --version
+        poetry --version
+
+        echo === Install dependencies for tests ===
+        poetry install
+
+        echo === Running unit tests ===
+        poetry run poetry run pytest -q --maxfail=1 --disable-warnings --cov=src/$(PKG) --cov-report term-missing --cov-fail-under=80 | tee coverage-report.txt
+        if grep -q "FAIL Required test coverage" coverage-report.txt; then
+          echo "ERROR: Target coverage was not reached, stopping pipeline."
+          exit 1
+        fi
+
+        echo === Looking up config from parameter store ===
+        dev_config=$(gcloud parametermanager parameters versions describe $_ENV_NAME --parameter=infra-test-config --location=global --project $PROJECT_ID --format=json | python3 -c "import sys, json; print(json.load(sys.stdin)['payload']['data'])" | base64 --decode)
+        THEMES_UI_IMAGE=$(echo $dev_config | python3 -c "import sys, json; print(json.load(sys.stdin)['cr-themes-ui-image'])")
+        TARGET_PROJECT_ID=$(echo $dev_config | python3 -c "import sys, json; print(json.load(sys.stdin)['project-id'])")
+
+        echo === Build Docker image, tag and push to GAR ===
+        docker build -t $$THEMES_UI_IMAGE:$SHORT_SHA -t $$THEMES_UI_IMAGE:latest .
+        docker push $$THEMES_UI_IMAGE:latest
+        docker push $$THEMES_UI_IMAGE:$SHORT_SHA
+
+        echo === Deploy themes UI ===
+        gcloud run deploy themes-ui --image=$$THEMES_UI_IMAGE:$SHORT_SHA --region $LOCATION --project $$TARGET_PROJECT_ID
+
+    id: Run unit tests, build, tag and deploy themes cloud run job
+    entrypoint: bash
+timeout: 1800s
+options:
+  logging: CLOUD_LOGGING_ONLY

cicd/cloudbuild_promote_preprod.yaml

@@ -0,0 +1,29 @@
+steps:
+  - name: gcr.io/$PROJECT_ID/testrunner
+    args:
+      - '-c'
+      - |
+        set -e
+        echo === Looking up config from parameter store ===
+        dev_config=$(gcloud parametermanager parameters versions describe dev --parameter=infra-test-config --location=global --project $PROJECT_ID --format=json | python3 -c "import sys, json; print(json.load(sys.stdin)['payload']['data'])" | base64 --decode)
+        THEMES_UI_IMAGE_DEV=$(echo $dev_config | python3 -c "import sys, json; print(json.load(sys.stdin)['cr-themes-ui-image'])")
+
+        preprod_config=$(gcloud parametermanager parameters versions describe preprod --parameter=infra-test-config --location=global --project $PROJECT_ID --format=json | python3 -c "import sys, json; print(json.load(sys.stdin)['payload']['data'])" | base64 --decode)
+        THEMES_UI_IMAGE_RELEASES=$(echo $preprod_config | python3 -c "import sys, json; print(json.load(sys.stdin)['cr-themes-ui-image'])")
+        TARGET_PROJECT_ID=$(echo $preprod_config | python3 -c "import sys, json; print(json.load(sys.stdin)['project-id'])")
+
+        echo === Pulling image dev image with tag $SHORT_SHA ===
+        docker pull $$THEMES_UI_IMAGE_DEV:$SHORT_SHA
+
+        echo === Build Docker image, tag and push to GAR ===
+        docker tag $$THEMES_UI_IMAGE_DEV:$SHORT_SHA $$THEMES_UI_IMAGE_RELEASES:$TAG_NAME
+        docker push $$THEMES_UI_IMAGE_RELEASES:$TAG_NAME
+
+        echo === Deploy themes UI ===
+        gcloud run deploy themes-ui --image=$$THEMES_UI_IMAGE_RELEASES:$TAG_NAME --region $LOCATION --project $$TARGET_PROJECT_ID
+
+    id: Tag and push release image
+    entrypoint: bash
+timeout: 2000s
+options:
+  logging: CLOUD_LOGGING_ONLY

pyproject.toml

@@ -14,6 +14,7 @@ google-cloud-storage = "^3.9.0"
 pyyaml = "^6.0.3"
 google-cloud-workflows = "^1.21.0"
 more-itertools = "^11.0.1"
+survey-assist-pii = { git = "https://github.com/ONSdigital/survey-assist-pii.git" }
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"

run_cicd_themes_to_sandbox.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This script is only intended for use in the Sandbox environment.
+#
+# Please set the environment variable CICD_PROJECT_ID i.e. export CICD_PROJECT_ID=
+
+if [[ ! -v CICD_PROJECT_ID ]]; then
+   echo "Please set the environment variable CICD_PROJECT_ID i.e. export CICD_PROJECT_ID="
+   exit 1
+fi
+
+ENV_NAME=sandbox # Sandbox use only
+
+GIT_SHA=$(git rev-parse --short HEAD)
+sandbox_config=$(gcloud parametermanager parameters versions describe $ENV_NAME --parameter=infra-test-config --location=global --project $CICD_PROJECT_ID --format=json | python3 -c "import sys, json; print(json.load(sys.stdin)['payload']['data'])" | base64 --decode)
+PROJECT_ID=$(echo $sandbox_config | python3 -c "import sys, json; print(json.load(sys.stdin)['project-id'])")
+CICD_SA=$(echo $sandbox_config | python3 -c "import sys, json; print(json.load(sys.stdin)['cicd-sa-email'])")
+REGION=$(echo $sandbox_config | python3 -c "import sys, json; print(json.load(sys.stdin)['region'])")
+CB_BUCKET=gs://${PROJECT_ID}_cloudbuild/soc-api
+
+gcloud beta builds submit . --config=cicd/cloudbuild_dev_and_sandbox.yaml \
+	--project $CICD_PROJECT_ID \
+	--service-account projects/$CICD_PROJECT_ID/serviceAccounts/$CICD_SA \
+	--gcs-source-staging-dir $CB_BUCKET \
+	--substitutions=SHORT_SHA=$GIT_SHA,_ENV_NAME=$ENV_NAME \
+	--region $REGION

src/theme_analysis_ui/app_templates/cancel.html

@@ -0,0 +1,44 @@
+{% extends "core.html" %}
+{% from "components/button/_macro.njk" import onsButton %}
+
+{% set containerClasses = "ons-container--full-width" %}
+{% set page_title = "Upload cancelled" %}
+
+{% block pageContent %}
+<div class="ons-page__container ons-container ons-u-pt-m ons-u-pb-l">
+    <main id="main-content" class="ons-grid ons-grid--column@xxs@s">
+        <div class="ons-grid__col ons-col-12 ons-col@l-8">
+
+            <h1 class="ons-u-fs-l ons-u-mb-m">
+                Upload cancelled
+            </h1>
+
+            <p class="ons-u-mb-m">
+                Cancelled due to potential sensitive information disclosure.
+            </p>
+
+            {% if pii_report_location %}
+                <p class="ons-u-mb-s">
+                    PII report is downloaded here:
+                </p>
+
+                <p class="ons-u-mb-l">
+                    <code>{{ pii_report_location }}</code>
+                </p>
+            {% else %}
+                <p class="ons-u-mb-l">
+                    No PII report location is available.
+                </p>
+            {% endif %}
+
+            <div class="ons-btn-group">
+                {{ onsButton({
+                    "text": "Return home",
+                    "url": url_for('ui.index')
+                }) }}
+            </div>
+
+        </div>
+    </main>
+</div>
+{% endblock %}

src/theme_analysis_ui/app_templates/review_responses.html

@@ -0,0 +1,81 @@
+{% extends "core.html" %}
+{% from "components/button/_macro.njk" import onsButton %}
+{% from "components/checkboxes/_macro.njk" import onsCheckboxes %}
+
+{% set containerClasses = "ons-container--full-width" %}
+{% set page_title = "Review responses" %}
+
+{% block pageContent %}
+<div class="ons-page__container ons-container ons-u-pt-m ons-u-pb-l">
+    <main id="main-content" class="ons-grid ons-grid--column@xxs@s">
+        <div class="ons-grid__col ons-col-12 ons-col@l-8">
+            <h2 class="ons-u-mb-m">
+                The following rows may disclose information
+            </h2>
+
+            {% if flagged_rows %}
+                <div class="ons-table__container ons-u-mb-m">
+                    <table class="ons-table">
+                        <thead class="ons-table__head">
+                            <tr class="ons-table__row">
+                                <th class="ons-table__header" scope="col">Row</th>
+                                <th class="ons-table__header" scope="col">Response</th>
+                            </tr>
+                        </thead>
+                        <tbody class="ons-table__body">
+                            {% for row in flagged_rows %}
+                                <tr class="ons-table__row">
+                                    <td class="ons-table__cell">{{ row.row_number }}</td>
+                                    <td class="ons-table__cell">{{ row.response_text }}</td>
+                                </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+                </div>
+            {% else %}
+                <p class="ons-u-mb-m">No potentially disclosive rows were found.</p>
+            {% endif %}
+
+            <form method="post" action="{{ url_for('ui.review_responses') }}">
+                <div class="ons-u-mb-l">
+                    {{
+                        onsCheckboxes({
+                            "id": "ignore-disclosure-warning",
+                            "legend": "Disclosure acknowledgement",
+                            "legendClasses": "ons-u-vh",
+                            "error": {
+                                "text": errors[0]
+                            } if errors else None,
+                            "checkboxes": [
+                                {
+                                    "id": "ignore-warning",
+                                    "name": "ignore_disclosure_warning",
+                                    "label": {
+                                        "text": "These are ok to ignore"
+                                    },
+                                    "value": "true"
+                                }
+                            ]
+                        })
+                    }}
+                </div>
+
+                <div class="ons-btn-group">
+                    {{ onsButton({
+                        "text": "Cancel",
+                        "url": url_for('ui.cancel'),
+                        "variants": "secondary"
+                    }) }}
+
+                    {{ onsButton({
+                        "text": "Continue",
+                        "type": "submit",
+                        "id": "continue-button"
+                    }) }}
+                </div>
+            </form>
+
+        </div>
+    </main>
+</div>
+{% endblock %}