feat(ci): parallelize deployments and readiness checks for showcase and showcase-rbac

.ci/pipelines/jobs/ocp-nightly.sh

@@ -46,9 +46,11 @@ handle_ocp_nightly() {
 
 run_standard_deployment_tests() {
   local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
-  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE}" "${url}"
   local rbac_url="https://${RELEASE_NAME_RBAC}-developer-hub-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
-  testing::check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${PW_PROJECT_SHOWCASE_RBAC}" "${rbac_url}"
+
+  testing::parallel_check_and_test \
+    "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE}" "${url}" \
+    "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${PW_PROJECT_SHOWCASE_RBAC}" "${rbac_url}"
 }
 
 run_runtime_config_change_tests() {

.ci/pipelines/jobs/ocp-pull.sh

@@ -23,8 +23,11 @@ handle_ocp_pull() {
   cluster_setup_ocp_helm
   initiate_deployments "${PW_PROJECT_SHOWCASE}" "${PW_PROJECT_SHOWCASE_RBAC}"
   deploy_test_backstage_customization_provider "${NAME_SPACE}"
+
   local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}"
-  testing::check_and_test "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE}" "${url}"
   local rbac_url="https://${RELEASE_NAME_RBAC}-developer-hub-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}"
-  testing::check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${PW_PROJECT_SHOWCASE_RBAC}" "${rbac_url}"
+
+  testing::parallel_check_and_test \
+    "${RELEASE_NAME}" "${NAME_SPACE}" "${PW_PROJECT_SHOWCASE}" "${url}" \
+    "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC}" "${PW_PROJECT_SHOWCASE_RBAC}" "${rbac_url}"
 }

.ci/pipelines/lib/testing.sh

@@ -61,42 +61,59 @@ testing::run_tests() {
   local e2e_tests_dir
   e2e_tests_dir=$(pwd)
 
-  yarn install --immutable > /tmp/yarn.install.log.txt 2>&1
+  # Use per-project paths for all outputs to allow parallel test execution
+  local project_logfile="${LOGFILE}-${artifacts_subdir}"
+  local project_test_results="${e2e_tests_dir}/test-results-${artifacts_subdir}"
+  local project_junit="junit-results-${artifacts_subdir}.xml"
+  local project_pw_report="${e2e_tests_dir}/playwright-report-${artifacts_subdir}"
+
+  yarn install --immutable > "/tmp/yarn.install.${artifacts_subdir}.log.txt" 2>&1
   local install_status=$?
   if [[ $install_status -ne 0 ]]; then
     log::error "=== YARN INSTALL FAILED ==="
-    cat /tmp/yarn.install.log.txt
+    cat "/tmp/yarn.install.${artifacts_subdir}.log.txt"
     exit $install_status
   else
     log::success "Yarn install completed successfully."
   fi
 
   yarn playwright install chromium
 
-  Xvfb :99 &
-  export DISPLAY=:99
+  # Reuse existing Xvfb display if already running (e.g. in parallel test execution).
+  # Only start a new Xvfb instance if DISPLAY is not set.
+  local xvfb_pid=""
+  if [[ -z "${DISPLAY:-}" ]]; then
+    Xvfb :99 &
+    xvfb_pid=$!
+    export DISPLAY=:99
+  fi
 
   (
     set -e
     log::info "Using PR container image: ${TAG_NAME}"
-    yarn playwright test --project="${playwright_project}"
-  ) 2>&1 | tee "/tmp/${LOGFILE}"
+    JUNIT_RESULTS="${project_junit}" \
+      PLAYWRIGHT_HTML_REPORT="${project_pw_report}" \
+      yarn playwright test --project="${playwright_project}" --output="${project_test_results}"
+  ) 2>&1 | tee "/tmp/${project_logfile}"
 
   local test_result=${PIPESTATUS[0]}
 
-  pkill Xvfb || true
+  # Only kill the Xvfb instance we started (not shared instances from the parent)
+  if [[ -n "${xvfb_pid}" ]]; then
+    kill "${xvfb_pid}" 2> /dev/null || true
+  fi
 
   # Use artifacts_subdir for artifact directory to keep artifacts organized
-  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/test-results/" "test-results" || true
-  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/${JUNIT_RESULTS}" || true
+  common::save_artifact "${artifacts_subdir}" "${project_test_results}/" "test-results" || true
+  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/${project_junit}" || true
   if [[ "${CI}" == "true" ]]; then
-    rsync "${ARTIFACT_DIR}/${artifacts_subdir}/${JUNIT_RESULTS}" "${SHARED_DIR}/junit-results-${artifacts_subdir}.xml" || true
+    rsync "${ARTIFACT_DIR}/${artifacts_subdir}/${project_junit}" "${SHARED_DIR}/junit-results-${artifacts_subdir}.xml" || true
   fi
 
   common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/screenshots/" "attachments/screenshots" || true
-  ansi2html < "/tmp/${LOGFILE}" > "/tmp/${LOGFILE}.html"
-  common::save_artifact "${artifacts_subdir}" "/tmp/${LOGFILE}.html" || true
-  common::save_artifact "${artifacts_subdir}" "${e2e_tests_dir}/playwright-report/" || true
+  ansi2html < "/tmp/${project_logfile}" > "/tmp/${project_logfile}.html"
+  common::save_artifact "${artifacts_subdir}" "/tmp/${project_logfile}.html" || true
+  common::save_artifact "${artifacts_subdir}" "${project_pw_report}/" || true
 
   echo "Playwright project '${playwright_project}' in namespace '${namespace}' (artifacts: ${artifacts_subdir}) RESULT: ${test_result}"
   local test_passed="true"
@@ -110,19 +127,60 @@ testing::run_tests() {
   local failed_tests
   if [[ "${test_result}" -eq 0 ]]; then
     failed_tests="0"
-  elif [[ -f "${e2e_tests_dir}/${JUNIT_RESULTS}" ]]; then
-    failed_tests=$(grep -oP 'failures="\K[0-9]+' "${e2e_tests_dir}/${JUNIT_RESULTS}" | head -n 1)
+  elif [[ -f "${e2e_tests_dir}/${project_junit}" ]]; then
+    failed_tests=$(grep -oP 'failures="\K[0-9]+' "${e2e_tests_dir}/${project_junit}" | head -n 1)
     failed_tests="${failed_tests:-some}"
     echo "Number of failed tests: ${failed_tests}"
   else
-    echo "JUnit results file not found: ${e2e_tests_dir}/${JUNIT_RESULTS}"
+    echo "JUnit results file not found: ${e2e_tests_dir}/${project_junit}"
     failed_tests="some"
     echo "Number of failed tests unknown, saving as $failed_tests."
   fi
   test_run_tracker::mark_test_result "$test_passed" "${failed_tests}"
   return "$test_result"
 }
 
+# Overlap readiness waits, then run tests sequentially.
+# Parallelizing the readiness waits (~5 min each) saves significant time while
+# running tests sequentially avoids OOM from concurrent Playwright instances.
+# After the parallel wait, check_and_test re-checks readiness (instant if already up).
+# Hardcoded for the base + RBAC deployment pair.
+# Args:
+#   $1-$4: base (release_name namespace playwright_project url)
+#   $5-$8: RBAC (release_name namespace playwright_project url)
+testing::parallel_check_and_test() {
+  local base_release=$1
+  local base_namespace=$2
+  local base_project=$3
+  local base_url=$4
+  local rbac_release=$5
+  local rbac_namespace=$6
+  local rbac_project=$7
+  local rbac_url=$8
+
+  # Phase 1: Wait for both deployments to be ready in parallel (lightweight HTTP polling)
+  log::section "Waiting for both deployments to be ready in parallel"
+
+  testing::check_backstage_running "${base_release}" "${base_namespace}" "${base_url}" "${base_project}" &
+  local base_ready_pid=$!
+  log::info "Base readiness check started in background (PID: ${base_ready_pid})"
+
+  testing::check_backstage_running "${rbac_release}" "${rbac_namespace}" "${rbac_url}" "${rbac_project}" &
+  local rbac_ready_pid=$!
+  log::info "RBAC readiness check started in background (PID: ${rbac_ready_pid})"
+
+  wait "${base_ready_pid}" || true
+  wait "${rbac_ready_pid}" || true
+
+  # Phase 2: Run tests sequentially to avoid OOM from concurrent Playwright instances.
+  # Re-uses check_and_test which re-checks readiness (instant if deployment is already up).
+  log::section "Running tests sequentially"
+
+  testing::check_and_test "${base_release}" "${base_namespace}" "${base_project}" "${base_url}"
+  testing::check_and_test "${rbac_release}" "${rbac_namespace}" "${rbac_project}" "${rbac_url}"
+  return 0
+}
+
 # ==============================================================================
 # Health Checks
 # ==============================================================================
@@ -185,7 +243,7 @@ testing::check_backstage_running() {
           || oc logs deployment/${release_name} -n "${namespace}" --tail=100 --all-containers=true 2> /dev/null || true
         log::error "Recent events:"
         oc get events -n "${namespace}" --sort-by='.lastTimestamp' | tail -20
-        common::save_artifact "${artifacts_subdir}" "/tmp/${LOGFILE}" || true
+        common::save_artifact "${artifacts_subdir}" "/tmp/${LOGFILE}-${artifacts_subdir}" || true
         return 1
       fi
 

.ci/pipelines/utils.sh

@@ -83,16 +83,17 @@ retrieve_pod_logs() {
   local pod_name=$1
   local container=$2
   local namespace=$3
-  local log_timeout=${4:-5}
-  local max_retries=${5:-3}
-  local backoff=${6:-2}
+  local pod_logs_dir=$4
+  local log_timeout=${5:-5}
+  local max_retries=${6:-3}
+  local backoff=${7:-2}
 
   log::debug "Retrieving logs for container: $container"
 
   # Retry with backoff for transient kubectl failures
   local attempt
   for ((attempt = 1; attempt <= max_retries; attempt++)); do
-    if timeout "${log_timeout}" kubectl logs "$pod_name" -c "$container" -n "$namespace" > "pod_logs/${pod_name}_${container}.log" 2> /dev/null; then
+    if timeout "${log_timeout}" kubectl logs "$pod_name" -c "$container" -n "$namespace" > "${pod_logs_dir}/${pod_name}_${container}.log" 2> /dev/null; then
       break
     fi
     if ((attempt == max_retries)); then
@@ -102,9 +103,9 @@ retrieve_pod_logs() {
     fi
   done
 
-  timeout "${log_timeout}" kubectl logs "$pod_name" -c "$container" -n "$namespace" --previous > "pod_logs/${pod_name}_${container}-previous.log" 2> /dev/null || {
+  timeout "${log_timeout}" kubectl logs "$pod_name" -c "$container" -n "$namespace" --previous > "${pod_logs_dir}/${pod_name}_${container}-previous.log" 2> /dev/null || {
     log::debug "Previous logs for container $container not found or timed out"
-    rm -f "pod_logs/${pod_name}_${container}-previous.log"
+    rm -f "${pod_logs_dir}/${pod_name}_${container}-previous.log"
   }
 }
 
@@ -113,18 +114,19 @@ retrieve_pod_logs() {
 _retrieve_all_logs_for_pod() {
   local pod_name=$1
   local namespace=$2
+  local pod_logs_dir=$3
   log::debug "Retrieving logs for pod: $pod_name in namespace $namespace"
 
   local init_containers
   init_containers=$(kubectl get pod "$pod_name" -n "$namespace" -o jsonpath='{.spec.initContainers[*].name}' 2> /dev/null)
   for init_container in $init_containers; do
-    retrieve_pod_logs "$pod_name" "$init_container" "$namespace"
+    retrieve_pod_logs "$pod_name" "$init_container" "$namespace" "$pod_logs_dir"
   done
 
   local containers
   containers=$(kubectl get pod "$pod_name" -n "$namespace" -o jsonpath='{.spec.containers[*].name}' 2> /dev/null)
   for container in $containers; do
-    retrieve_pod_logs "$pod_name" "$container" "$namespace"
+    retrieve_pod_logs "$pod_name" "$container" "$namespace" "$pod_logs_dir"
   done
   return 0
 }
@@ -133,7 +135,10 @@ save_all_pod_logs() {
   set +e
   local namespace=$1
   local artifacts_subdir="${2:-$namespace}"
-  rm -rf pod_logs && mkdir -p pod_logs
+
+  # Use a unique temp directory per artifacts_subdir to avoid conflicts
+  local pod_logs_dir="/tmp/pod_logs-${artifacts_subdir}"
+  rm -rf "${pod_logs_dir}" && mkdir -p "${pod_logs_dir}"
 
   local pod_names
   if ! pod_names=$(kubectl get pods -n "$namespace" -o jsonpath='{.items[*].metadata.name}' 2> /dev/null); then
@@ -145,7 +150,7 @@ save_all_pod_logs() {
   # Gather logs from all pods in parallel
   local pids=()
   for pod_name in $pod_names; do
-    _retrieve_all_logs_for_pod "$pod_name" "$namespace" &
+    _retrieve_all_logs_for_pod "$pod_name" "$namespace" "$pod_logs_dir" &
     pids+=($!)
   done
 
@@ -154,7 +159,7 @@ save_all_pod_logs() {
     wait "$pid" 2> /dev/null || true
   done
 
-  common::save_artifact "${artifacts_subdir}" "pod_logs/" "pod_logs" || true
+  common::save_artifact "${artifacts_subdir}" "${pod_logs_dir}/" "pod_logs" || true
   set -e
 }
 
@@ -625,8 +630,23 @@ initiate_deployments() {
   local rbac_artifacts_subdir=$2
 
   cd "${DIR}"
-  base_deployment "${base_artifacts_subdir}"
-  rbac_deployment "${rbac_artifacts_subdir}"
+  log::section "Starting parallel deployments (base + RBAC)"
+
+  base_deployment "${base_artifacts_subdir}" &
+  local base_pid=$!
+  rbac_deployment "${rbac_artifacts_subdir}" &
+  local rbac_pid=$!
+
+  local failed=0
+  wait "${base_pid}" || {
+    log::error "Base deployment failed"
+    failed=1
+  }
+  wait "${rbac_pid}" || {
+    log::error "RBAC deployment failed"
+    failed=1
+  }
+  return "${failed}"
 }
 
 # OSD-GCP specific deployment functions that merge diff files and skip orchestrator workflows
@@ -693,8 +713,23 @@ initiate_deployments_osd_gcp() {
   local rbac_artifacts_subdir=$2
 
   cd "${DIR}"
-  base_deployment_osd_gcp "${base_artifacts_subdir}"
-  rbac_deployment_osd_gcp "${rbac_artifacts_subdir}"
+  log::section "Starting parallel OSD-GCP deployments (base + RBAC)"
+
+  base_deployment_osd_gcp "${base_artifacts_subdir}" &
+  local base_pid=$!
+  rbac_deployment_osd_gcp "${rbac_artifacts_subdir}" &
+  local rbac_pid=$!
+
+  local failed=0
+  wait "${base_pid}" || {
+    log::error "OSD-GCP base deployment failed"
+    failed=1
+  }
+  wait "${rbac_pid}" || {
+    log::error "OSD-GCP RBAC deployment failed"
+    failed=1
+  }
+  return "${failed}"
 }
 
 # install base RHDH deployment before upgrade