ci: wpb 25526 auto run upgrade test (wpb-25526)

.github/workflows/qa-android-critical-flow-tests.yml

@@ -23,13 +23,13 @@ on:
         type: string
 
       isUpgrade:
-        description: "Upgrade test? If true, oldBuildNumber is REQUIRED."
+        description: "Upgrade test? If true, oldBuildNumber is optional and defaults to the previous APK."
         required: true
         default: false
         type: boolean
 
       oldBuildNumber:
-        description: "For upgrade runs: type the old build number here"
+        description: "For upgrade runs: optional old build number. Leave empty to auto-pick the previous APK."
         required: false
         default: ""
         type: string
@@ -138,7 +138,7 @@ jobs:
 
           {
             echo "appBuildNumber=latest"
-            echo "isUpgrade=false"
+            echo "isUpgrade=true"
             echo "oldBuildNumber="
             echo "enforceAppInstall=false"
             echo "flavor=internal release candidate"
@@ -297,7 +297,53 @@ jobs:
           RERUN_FAILED_ENABLED: ${{ needs.validate-and-resolve-inputs.outputs.resolvedRerunFailedEnabled }}
           RERUN_FAILED_COUNT: ${{ needs.validate-and-resolve-inputs.outputs.resolvedRerunFailedCount }}
           ALLURE_RESULTS_ROOT: ${{ runner.temp }}/allure-results
-        run: bash scripts/qa_android_ui_tests/run_ui_tests.sh
+        run: |
+          if [[ "${IS_UPGRADE}" == "true" && ( "${RESOLVED_CATEGORY}" == "upgrade" || "${RESOLVED_TESTCASE_ID}" == "TC-8607" ) ]]; then
+            # Single upgrade-tagged runs do not need the normal phase. They only
+            # pin execution to one device and install the old APK before launch.
+            upgrade_device="${DEVICE_LIST%% *}"
+            DEVICE_LIST="${upgrade_device}" \
+              DEVICE_COUNT="1" \
+              APP_APK_BEFORE_ATTEMPT="${OLD_APK_DEVICE_PATH}" \
+              RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-upgrade" \
+              bash scripts/qa_android_ui_tests/run_ui_tests.sh
+          elif [[ "${IS_UPGRADE}" == "true" && -z "${RESOLVED_TESTCASE_ID}" ]]; then
+            # For broad upgrade runs, run the selected normal tests on the new APK first.
+            # Then run only upgrade tests afterward, starting them from the old APK.
+            upgrade_device="${DEVICE_LIST%% *}"
+            normal_category="${RESOLVED_CATEGORY}"
+            normal_status=0
+            upgrade_status=0
+
+            echo "Running normal tests first on the new APK, excluding upgrade tests."
+            RESOLVED_CATEGORY="${normal_category}" \
+              EXCLUDE_CATEGORY="upgrade" \
+              RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-normal" \
+              ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/normal" \
+              bash scripts/qa_android_ui_tests/run_ui_tests.sh || normal_status=$?
+
+            echo "Running upgrade tests last with old APK installed before each attempt."
+            DEVICE_LIST="${upgrade_device}" \
+              DEVICE_COUNT="1" \
+              RESOLVED_CATEGORY="upgrade" \
+              APP_APK_BEFORE_ATTEMPT="${OLD_APK_DEVICE_PATH}" \
+              RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-upgrade" \
+              ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/upgrade" \
+              bash scripts/qa_android_ui_tests/run_ui_tests.sh || upgrade_status=$?
+
+            RETRY_STATE_DIRS="${RUNNER_TEMP}/retry-state-normal ${RUNNER_TEMP}/retry-state-upgrade" \
+              COMBINED_RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-combined" \
+              bash scripts/qa_android_ui_tests/reporting.sh combine-retry-state
+
+            if (( normal_status != 0 )); then
+              exit "${normal_status}"
+            fi
+            if (( upgrade_status != 0 )); then
+              exit "${upgrade_status}"
+            fi
+          else
+            bash scripts/qa_android_ui_tests/run_ui_tests.sh
+          fi
 
       # Export one standard artifact so future manual deflake runs can reuse
       # this workflow's exact run context and leftover failed test list.
@@ -320,7 +366,7 @@ jobs:
           RESOLVED_CATEGORY: ${{ needs.validate-and-resolve-inputs.outputs.resolvedCategory }}
           APP_BUILD_NUMBER_INPUT: ${{ needs.validate-and-resolve-inputs.outputs.resolvedAppBuildNumber }}
           IS_UPGRADE: ${{ needs.validate-and-resolve-inputs.outputs.resolvedIsUpgrade }}
-          OLD_BUILD_NUMBER: ${{ needs.validate-and-resolve-inputs.outputs.resolvedOldBuildNumber }}
+          OLD_BUILD_NUMBER: ${{ steps.download_apks.outputs.OLD_BUILD_NUMBER }}
           ENFORCE_APP_INSTALL: ${{ needs.validate-and-resolve-inputs.outputs.resolvedEnforceAppInstall }}
           TESTINY_RUN_NAME: ${{ needs.validate-and-resolve-inputs.outputs.resolvedTestinyRunName }}
           ANDROID_DEVICE_ID: ${{ needs.validate-and-resolve-inputs.outputs.resolvedAndroidDeviceId }}

.github/workflows/qa-android-ui-test-manual-deflake.yml

@@ -166,7 +166,51 @@ jobs:
           RERUN_FAILED_ENABLED: ${{ steps.validate_deflake_input.outputs.rerunFailedEnabled }}
           RERUN_FAILED_COUNT: ${{ steps.validate_deflake_input.outputs.rerunFailedCount }}
           ALLURE_RESULTS_ROOT: ${{ runner.temp }}/allure-results
-        run: bash scripts/qa_android_ui_tests/run_ui_tests.sh
+        run: |
+          if [[ "${IS_UPGRADE}" == "true" ]]; then
+            upgrade_failed_file="${RUNNER_TEMP}/deflake-upgrade-failed-tests.txt"
+            normal_failed_file="${RUNNER_TEMP}/deflake-normal-failed-tests.txt"
+            normal_status=0
+            upgrade_status=0
+            retry_state_dirs=""
+            grep 'criticalFlows\.UpgradeVersion#' "${INITIAL_FAILED_TESTS_FILE}" > "${upgrade_failed_file}" || true
+            grep -v 'criticalFlows\.UpgradeVersion#' "${INITIAL_FAILED_TESTS_FILE}" > "${normal_failed_file}" || true
+
+            if [[ -s "${normal_failed_file}" ]]; then
+              INITIAL_FAILED_TESTS_FILE="${normal_failed_file}" \
+                RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-normal-deflake" \
+                ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/normal" \
+                bash scripts/qa_android_ui_tests/run_ui_tests.sh || normal_status=$?
+              retry_state_dirs="${retry_state_dirs} ${RUNNER_TEMP}/retry-state-normal-deflake"
+            fi
+
+            if [[ -s "${upgrade_failed_file}" ]]; then
+              upgrade_device="${DEVICE_LIST%% *}"
+              DEVICE_LIST="${upgrade_device}" \
+                DEVICE_COUNT="1" \
+                INITIAL_FAILED_TESTS_FILE="${upgrade_failed_file}" \
+                APP_APK_BEFORE_ATTEMPT="${OLD_APK_DEVICE_PATH}" \
+                RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-upgrade-deflake" \
+                ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/upgrade" \
+                bash scripts/qa_android_ui_tests/run_ui_tests.sh || upgrade_status=$?
+              retry_state_dirs="${retry_state_dirs} ${RUNNER_TEMP}/retry-state-upgrade-deflake"
+            fi
+
+            if [[ -n "${retry_state_dirs// /}" ]]; then
+              RETRY_STATE_DIRS="${retry_state_dirs}" \
+                COMBINED_RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-deflake-combined" \
+                bash scripts/qa_android_ui_tests/reporting.sh combine-retry-state
+            fi
+
+            if (( normal_status != 0 )); then
+              exit "${normal_status}"
+            fi
+            if (( upgrade_status != 0 )); then
+              exit "${upgrade_status}"
+            fi
+          else
+            bash scripts/qa_android_ui_tests/run_ui_tests.sh
+          fi
 
       # Publish a fresh deflake bundle so a later run can deflake this manual
       # deflake execution without going back to qa-android-critical-flow-tests.yml.
@@ -189,7 +233,7 @@ jobs:
           RESOLVED_CATEGORY: ${{ steps.validate_deflake_input.outputs.selectorType == 'category' && steps.validate_deflake_input.outputs.selectorValue || '' }}
           APP_BUILD_NUMBER_INPUT: ${{ steps.validate_deflake_input.outputs.resolvedBuildNumber || steps.validate_deflake_input.outputs.appBuildNumberInput }}
           IS_UPGRADE: ${{ steps.validate_deflake_input.outputs.isUpgrade }}
-          OLD_BUILD_NUMBER: ${{ steps.validate_deflake_input.outputs.oldBuildNumber }}
+          OLD_BUILD_NUMBER: ${{ steps.download_apks.outputs.OLD_BUILD_NUMBER }}
           ENFORCE_APP_INSTALL: ${{ steps.validate_deflake_input.outputs.enforceAppInstall }}
           TESTINY_RUN_NAME: ${{ steps.validate_deflake_input.outputs.effectiveTestinyRunName }}
           ANDROID_DEVICE_ID: ${{ steps.validate_deflake_input.outputs.androidDeviceId }}

scripts/qa_android_ui_tests/execution_setup.sh

@@ -92,6 +92,8 @@ download_apks() {
 
   local new_s3_key=""
   local old_s3_key=""
+  local new_apk_name=""
+  local old_apk_name=""
   while IFS= read -r line || [[ -n "${line}" ]]; do
     [[ -z "${line}" ]] && continue
     if [[ "${line}" != *=* ]]; then
@@ -119,6 +121,12 @@ download_apks() {
       OLD_S3_KEY)
         old_s3_key="${value}"
         ;;
+      NEW_APK_NAME)
+        new_apk_name="${value}"
+        ;;
+      OLD_APK_NAME)
+        old_apk_name="${value}"
+        ;;
     esac
   done < "${apk_env_file}"
 
@@ -133,6 +141,9 @@ download_apks() {
   test -s "${new_apk_path}"
 
   if [[ "${IS_UPGRADE:-}" == "true" ]]; then
+    echo "OLD_APK_NAME=${old_apk_name}"
+    echo "NEW_APK_NAME=${new_apk_name}"
+
     if [[ -z "${old_s3_key}" ]]; then
       echo "ERROR: Missing OLD_S3_KEY for upgrade flow"
       exit 1
@@ -198,6 +209,8 @@ install_apks_on_devices() {
   : "${NEW_APK_PATH:?NEW_APK_PATH missing}"
   : "${GITHUB_ENV:?GITHUB_ENV not set}"
 
+  # Export stable device paths so upgrade tests can install the new APK during
+  # the in-test upgrade step. CI also keeps the old APK available for setup.
   local new_apk_device_path="/data/local/tmp/Wire.new.apk"
   local old_apk_device_path="/data/local/tmp/Wire.old.apk"
   echo "NEW_APK_DEVICE_PATH=${new_apk_device_path}" >> "$GITHUB_ENV"
@@ -228,15 +241,15 @@ install_apks_on_devices() {
 
     if [[ "${IS_UPGRADE:-}" == "true" ]]; then
       : "${OLD_APK_PATH:?OLD_APK_PATH missing for upgrade}"
-      # Upgrade tests need both APKs on the device because instrumentation
-      # receives those paths and performs the in-test upgrade flow itself.
+      # Keep both APK files available on the device for the in-test upgrade
+      # flow. Remove stale copies first so retries do not reuse an older file.
       ${adb_cmd} shell rm -f "${new_apk_device_path}" "${old_apk_device_path}" || true
       ${adb_cmd} push "${OLD_APK_PATH}" "${old_apk_device_path}" >/dev/null
       ${adb_cmd} push "${NEW_APK_PATH}" "${new_apk_device_path}" >/dev/null
-      ${adb_cmd} install ${install_flags} "${OLD_APK_PATH}"
-    else
-      ${adb_cmd} install ${install_flags} "${NEW_APK_PATH}"
     fi
+    # Always install the selected new APK during general setup. The upgrade
+    # phase explicitly reinstalls the old device-side APK before instrumentation.
+    ${adb_cmd} install ${install_flags} "${NEW_APK_PATH}"
 
     if ! ${adb_cmd} shell pm list packages | grep -qx "package:${APP_ID}"; then
       echo "ERROR: '${APP_ID}' not installed on ${serial}."

scripts/qa_android_ui_tests/merge_allure_results.py

@@ -39,9 +39,10 @@ def contains_result_files(device_dir: Path) -> bool:
         src_dir = src_candidate if src_candidate.is_dir() else device_dir
         return src_dir.is_dir() and any(src_dir.glob("*-result.json"))
 
-    # Discover retry-aware layout first: OUT_DIR/attempt-N/<serial>/...
+    # Discover retry-aware layout first. Upgrade runs can keep phase results
+    # under OUT_DIR/<phase>/attempt-N/<serial>/..., so search recursively.
     attempt_dirs = []
-    for candidate in sorted(base_dir.iterdir()):
+    for candidate in sorted(base_dir.rglob("attempt-*")):
         if not candidate.is_dir():
             continue
         if not candidate.name.startswith("attempt-"):

scripts/qa_android_ui_tests/reporting.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 # Reporting and publication utilities for QA Android UI test workflows.
 
 usage() {
-  echo "Usage: $0 {remove-runtime-secrets|pull-allure-results|prepare-deflake-bundle|merge-allure-results|summarize-allure-results|generate-allure-report|publish-allure-report|cleanup-workspace}" >&2
+  echo "Usage: $0 {remove-runtime-secrets|pull-allure-results|combine-retry-state|prepare-deflake-bundle|merge-allure-results|summarize-allure-results|generate-allure-report|publish-allure-report|cleanup-workspace}" >&2
   exit 2
 }
 
@@ -22,12 +22,10 @@ pull_allure_results() {
   mkdir -p "${out_dir}"
 
   # Retry-aware runs already persist per-attempt results during execution.
-  if compgen -G "${out_dir}/attempt-*" >/dev/null; then
-    if find "${out_dir}"/attempt-* -type f -name '*-result.json' -print -quit | grep -q .; then
-      echo "Per-attempt Allure results already present under ${out_dir}; skipping fallback pull."
-      return
-    fi
-    echo "Attempt folders exist but no result files found yet; running fallback pull."
+  # Upgrade runs may store phase results one level deeper under this root.
+  if find "${out_dir}" -type f -name '*-result.json' -print -quit | grep -q .; then
+    echo "Per-attempt Allure results already present under ${out_dir}; skipping fallback pull."
+    return
   fi
 
   if [[ -z "${DEVICE_LIST:-}" ]]; then
@@ -47,6 +45,58 @@ pull_allure_results() {
   done
 }
 
+combine_retry_state() {
+  : "${RETRY_STATE_DIRS:?RETRY_STATE_DIRS not set}"
+  : "${COMBINED_RETRY_STATE_DIR:?COMBINED_RETRY_STATE_DIR not set}"
+
+  # Upgrade critical-flow runs produce separate retry states for the normal and
+  # upgrade phases. Merge them back into one contract for the deflake artifact.
+  mkdir -p "${COMBINED_RETRY_STATE_DIR}"
+  local first_failed_file="${COMBINED_RETRY_STATE_DIR}/first-attempt-failed.txt"
+  local final_failed_file="${COMBINED_RETRY_STATE_DIR}/final-failed.txt"
+  : > "${first_failed_file}"
+  : > "${final_failed_file}"
+
+  read -ra STATE_DIRS <<< "${RETRY_STATE_DIRS}"
+  for state_dir in "${STATE_DIRS[@]}"; do
+    [[ -d "${state_dir}" ]] || continue
+
+    if [[ -s "${state_dir}/first-attempt-failed.txt" ]]; then
+      cat "${state_dir}/first-attempt-failed.txt" >> "${first_failed_file}"
+    fi
+
+    local latest_failed_file=""
+    while IFS= read -r candidate; do
+      latest_failed_file="${candidate}"
+    done < <(find "${state_dir}" -maxdepth 1 -type f -name 'attempt-*-failed.txt' | sort -V)
+
+    if [[ -n "${latest_failed_file}" && -s "${latest_failed_file}" ]]; then
+      cat "${latest_failed_file}" >> "${final_failed_file}"
+    fi
+  done
+
+  sort -u "${first_failed_file}" -o "${first_failed_file}"
+  sort -u "${final_failed_file}" -o "${final_failed_file}"
+
+  local first_failed_count
+  local final_failed_count
+  local passed_on_rerun_count=0
+  first_failed_count="$(grep -cve '^[[:space:]]*$' "${first_failed_file}" || true)"
+  final_failed_count="$(grep -cve '^[[:space:]]*$' "${final_failed_file}" || true)"
+  if (( first_failed_count > final_failed_count )); then
+    passed_on_rerun_count=$((first_failed_count - final_failed_count))
+  fi
+
+  {
+    echo "RETRY_STATE_DIR=${COMBINED_RETRY_STATE_DIR}"
+    echo "FIRST_FAILED_TESTS_FILE=${first_failed_file}"
+    echo "FINAL_FAILED_TESTS_FILE=${final_failed_file}"
+    echo "FIRST_FAILED_TESTS_COUNT=${first_failed_count}"
+    echo "FINAL_FAILED_TESTS_COUNT=${final_failed_count}"
+    echo "PASSED_ON_RERUN_COUNT=${passed_on_rerun_count}"
+  } >> "${GITHUB_ENV}"
+}
+
 prepare_deflake_bundle() {
   : "${DEFLAKE_BUNDLE_DIR:?DEFLAKE_BUNDLE_DIR not set}"
 
@@ -237,6 +287,14 @@ cleanup_workspace() {
   rm -f "secrets.json" "${RUNNER_TEMP}/secrets.json" || true
   rm -f "${RUNNER_TEMP}/Wire.apk" "${RUNNER_TEMP}/Wire.old.apk" || true
 
+  if [[ -n "${DEVICE_LIST:-}" ]]; then
+    # Remove APK copies pushed for upgrade runs; each new job pushes fresh files.
+    read -ra DEVICES <<< "${DEVICE_LIST}"
+    for serial in "${DEVICES[@]}"; do
+      adb -s "${serial}" shell rm -f /data/local/tmp/Wire.old.apk /data/local/tmp/Wire.new.apk || true
+    done
+  fi
+
   rm -rf "${ALLURE_RESULTS_DIR}" || true
   rm -rf "${ALLURE_RESULTS_MERGED_DIR}" || true
   rm -rf "${ALLURE_REPORT_DIR}" || true
@@ -255,6 +313,9 @@ case "${1:-}" in
   pull-allure-results)
     pull_allure_results
     ;;
+  combine-retry-state)
+    combine_retry_state
+    ;;
   prepare-deflake-bundle)
     prepare_deflake_bundle
     ;;