Skip to content

OTA-1800 Optimize retarget tests #72928

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Dec 26, 2025
Merged

OTA-1800 Optimize retarget tests #72928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions ...ts/openshift-verification-tests-main__ota-multi-stable-4.21-upgrade-from-stable-4.20.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ base_images:
namespace: ci
tag: latest
releases:
intermediate1:
intermediate:
prerelease:
architecture: multi
product: ocp
Expand Down Expand Up @@ -114,7 +114,7 @@ tests:
commands: echo "${INTERMEDIATE_RETARGET}" > ${SHARED_DIR}/upgrade-edge
dependencies:
- env: INTERMEDIATE_RETARGET
name: release:intermediate1
name: release:intermediate
from: cli
resources:
requests:
Expand All @@ -123,6 +123,13 @@ tests:
- ref: cucushift-upgrade-retarget
- ref: cucushift-upgrade-healthcheck
workflow: baremetal-lab-ipi
- as: gcp-ipi-retarget-f28
cron: 5 7 24 * *
steps:
cluster_profile: gcp-qe
test:
- ref: cucushift-upgrade-retarget
workflow: cucushift-installer-rehearse-gcp-ipi
zz_generated_metadata:
branch: main
org: openshift
Expand Down
11 changes: 9 additions & 2 deletions ...ts/openshift-verification-tests-main__ota-multi-stable-4.21-upgrade-from-stable-4.21.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ base_images:
namespace: ci
tag: latest
releases:
intermediate1:
intermediate:
prerelease:
architecture: multi
product: ocp
Expand Down Expand Up @@ -71,7 +71,7 @@ tests:
commands: echo "${INTERMEDIATE_RETARGET}" > ${SHARED_DIR}/upgrade-edge
dependencies:
- env: INTERMEDIATE_RETARGET
name: release:intermediate1
name: release:intermediate
from: cli
resources:
requests:
Expand All @@ -80,6 +80,13 @@ tests:
- ref: cucushift-upgrade-retarget
- ref: cucushift-upgrade-healthcheck
workflow: baremetal-lab-ipi
- as: gcp-ipi-retarget-f28
cron: 41 20 10 * *
steps:
cluster_profile: gcp-qe
test:
- ref: cucushift-upgrade-retarget
workflow: cucushift-installer-rehearse-gcp-ipi
zz_generated_metadata:
branch: main
org: openshift
Expand Down
148 changes: 148 additions & 0 deletions ...erator/jobs/openshift/verification-tests/openshift-verification-tests-main-periodics.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45984,6 +45984,80 @@ periodics:
- name: result-aggregator
secret:
secretName: result-aggregator
- agent: kubernetes
cluster: build02
cron: 5 7 24 * *
decorate: true
decoration_config:
skip_cloning: true
extra_refs:
- base_ref: main
org: openshift
repo: verification-tests
labels:
ci-operator.openshift.io/cloud: gcp
ci-operator.openshift.io/cloud-cluster-profile: gcp-qe
ci-operator.openshift.io/variant: ota-multi-stable-4.21-upgrade-from-stable-4.20
ci.openshift.io/generator: prowgen
pj-rehearse.openshift.io/can-be-rehearsed: "true"
name: periodic-ci-openshift-verification-tests-main-ota-multi-stable-4.21-upgrade-from-stable-4.20-gcp-ipi-retarget-f28
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --lease-server-credentials-file=/etc/boskos/credentials
- --report-credentials-file=/etc/report/credentials
- --secret-dir=/secrets/ci-pull-credentials
- --target=gcp-ipi-retarget-f28
- --variant=ota-multi-stable-4.21-upgrade-from-stable-4.20
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
imagePullPolicy: Always
name: ""
resources:
requests:
cpu: 10m
volumeMounts:
- mountPath: /etc/boskos
name: boskos
readOnly: true
- mountPath: /secrets/ci-pull-credentials
name: ci-pull-credentials
readOnly: true
- mountPath: /secrets/gcs
name: gcs-credentials
readOnly: true
- mountPath: /secrets/manifest-tool
name: manifest-tool-local-pusher
readOnly: true
- mountPath: /etc/pull-secret
name: pull-secret
readOnly: true
- mountPath: /etc/report
name: result-aggregator
readOnly: true
serviceAccountName: ci-operator
volumes:
- name: boskos
secret:
items:
- key: credentials
path: credentials
secretName: boskos-credentials
- name: ci-pull-credentials
secret:
secretName: ci-pull-credentials
- name: manifest-tool-local-pusher
secret:
secretName: manifest-tool-local-pusher
- name: pull-secret
secret:
secretName: registry-pull-credentials
- name: result-aggregator
secret:
secretName: result-aggregator
- agent: kubernetes
cluster: build09
cron: 53 21 13 * *
Expand Down Expand Up @@ -46059,3 +46133,77 @@ periodics:
- name: result-aggregator
secret:
secretName: result-aggregator
- agent: kubernetes
cluster: build02
cron: 41 20 10 * *
decorate: true
decoration_config:
skip_cloning: true
extra_refs:
- base_ref: main
org: openshift
repo: verification-tests
labels:
ci-operator.openshift.io/cloud: gcp
ci-operator.openshift.io/cloud-cluster-profile: gcp-qe
ci-operator.openshift.io/variant: ota-multi-stable-4.21-upgrade-from-stable-4.21
ci.openshift.io/generator: prowgen
pj-rehearse.openshift.io/can-be-rehearsed: "true"
name: periodic-ci-openshift-verification-tests-main-ota-multi-stable-4.21-upgrade-from-stable-4.21-gcp-ipi-retarget-f28
spec:
containers:
- args:
- --gcs-upload-secret=/secrets/gcs/service-account.json
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
- --lease-server-credentials-file=/etc/boskos/credentials
- --report-credentials-file=/etc/report/credentials
- --secret-dir=/secrets/ci-pull-credentials
- --target=gcp-ipi-retarget-f28
- --variant=ota-multi-stable-4.21-upgrade-from-stable-4.21
command:
- ci-operator
image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
imagePullPolicy: Always
name: ""
resources:
requests:
cpu: 10m
volumeMounts:
- mountPath: /etc/boskos
name: boskos
readOnly: true
- mountPath: /secrets/ci-pull-credentials
name: ci-pull-credentials
readOnly: true
- mountPath: /secrets/gcs
name: gcs-credentials
readOnly: true
- mountPath: /secrets/manifest-tool
name: manifest-tool-local-pusher
readOnly: true
- mountPath: /etc/pull-secret
name: pull-secret
readOnly: true
- mountPath: /etc/report
name: result-aggregator
readOnly: true
serviceAccountName: ci-operator
volumes:
- name: boskos
secret:
items:
- key: credentials
path: credentials
secretName: boskos-credentials
- name: ci-pull-credentials
secret:
secretName: ci-pull-credentials
- name: manifest-tool-local-pusher
secret:
secretName: manifest-tool-local-pusher
- name: pull-secret
secret:
secretName: registry-pull-credentials
- name: result-aggregator
secret:
secretName: result-aggregator
92 changes: 62 additions & 30 deletions ci-operator/step-registry/cucushift/upgrade/retarget/cucushift-upgrade-retarget-commands.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,26 @@ function clear_upgrade() {
fi
}

# Get multi pull spec for ***next minor version***, return 1 if failed to get to digest.
# $ get_multi_pullspec 4.20.1
function get_multi_pullspec() {
local major_version minor_version version="${1}"
major_version="$(echo "$version" | cut -f1 -d.)"
minor_version="$(echo "$version" | cut -f2 -d.)"
minor_version=$((minor_version + 1))

echo "Will get pullspec for ${major_version}.${minor_version}.0-ec.0-multi" >&2

digest=$(oc image info --show-multiarch quay.io/openshift-release-dev/ocp-release:"${major_version}"."${minor_version}".0-ec.0-multi -o json | jq -r ".[0].listDigest")
if [[ -n "$digest" ]]; then
echo "quay.io/openshift-release-dev/ocp-release@${digest}"
return 0
else
echo "Error: unable to get digest"
return 1
fi
}

# Wait upgrade to start, https://polarion.engineering.redhat.com/polarion/#/project/OSE/workitem?id=OCP-25473
# Following conditions will be used to determine if an upgrade is started
# Progressing=True
Expand Down Expand Up @@ -289,8 +309,11 @@ function upgrade() {
# *******************
# No matter y stream or z stream retarget, the TARGET always the final version we want to upgrade to
# To make OCP-25473 not block the whole pipeline, the upgrade orders are:
# if the minor version in ${SHARED_DIR}/upgrade-edge great than target, then
# if the minor version in ${SHARED_DIR}/upgrade-edge is greater than target, then
# initial -> target -> ${SHARED_DIR}/upgrade-edge
# if ${SHARED_DIR}/upgrade-edge is empty, then
# 1. get a new target version which greater than target
# 2. initial -> target -> new target
# if the minor version in ${SHARED_DIR}/upgrade-edge equal to target, then
# 1. we find a target+1 version as third target upgrade version
# 2. initial -> ${SHARED_DIR}/upgrade-edge -> target -> third target version
Expand All @@ -301,46 +324,55 @@ function upgrade() {
local testcase="OCP-25473"
export IMPLICIT_ENABLED_CASES="${IMPLICIT_ENABLED_CASES} ${testcase}"

intermediate_image="$(< "${SHARED_DIR}/upgrade-edge")"
if [[ -z "${intermediate_image:-}" ]]; then
echo "Error: The intermediate image is not given, break the job"
return 1
fi

echo "Prepare test data"
INTERMEDIATE_MINOR_VERSION="$(oc adm release info -ojson "$intermediate_image" | jq -r '.metadata.version' | cut -f2 -d.)"
if [[ "$INTERMEDIATE_MINOR_VERSION" -gt "$TARGET_MINOR_VERSION" ]]; then

if ! [ -e "${SHARED_DIR}/upgrade-edge" ]; then
# This is y stream retarget upgrade, e.g.: 4.y -> (4.y | 4.y+1) -> (4.y+1 | 4.y+2)
echo "upgrade-edge is empty, get a new target"
first_upgrade_to_image=${TARGET}
second_upgrade_to_image=${intermediate_image}
if ! second_upgrade_to_image=$(get_multi_pullspec "${TARGET_VERSION}"); then
echo "Error: OCP-25473 could not get new target pullsepc"
return 1
fi
echo "New target pullspec is: ${second_upgrade_to_image}"
block_second=true
else
if [[ "$INTERMEDIATE_MINOR_VERSION" == "$TARGET_MINOR_VERSION" ]]; then
# This is z stream retarget upgrade, e.g.: 4.y -> (4.y | 4.y+1) -> (4.y | 4.y+1)
first_upgrade_to_image=${intermediate_image}
second_upgrade_to_image=${TARGET}

if oc adm release info -o jsonpath='{.digest}' quay.io/openshift-release-dev/ocp-release:4.$((TARGET_MINOR_VERSION+1)).0-ec.0-x86_64 2>&1; then
# if third_upgrade_to_image not empty, we will retarget to it, but it will always be blocked
# this can make sure we always have z and y stream retarget in one job
latest_minor_version=$(( TARGET_MINOR_VERSION+1 ))
third_upgrade_to_image="$( oc adm release info -o jsonpath='{.digest}' quay.io/openshift-release-dev/ocp-release:4.${latest_minor_version}.0-ec.0-x86_64 )"
third_upgrade_to_image="quay.io/openshift-release-dev/ocp-release@${third_upgrade_to_image}"
fi

block_second=false
intermediate_image="$(< "${SHARED_DIR}/upgrade-edge")"
echo "upgrade-edge exists, the pullspec is: ${intermediate_image}"
INTERMEDIATE_MINOR_VERSION="$(oc adm release info -ojson "$intermediate_image" | jq -r '.metadata.version' | cut -f2 -d.)"
if [[ "$INTERMEDIATE_MINOR_VERSION" -gt "$TARGET_MINOR_VERSION" ]]; then
# This is y stream retarget upgrade, e.g.: 4.y -> (4.y | 4.y+1) -> (4.y+1 | 4.y+2)
first_upgrade_to_image=${TARGET}
second_upgrade_to_image=${intermediate_image}
block_second=true
else
# If INTERMEDIATE_MINOR_VERSION < TARGET_MINOR_VERSION, then we will naver be able to upgrade to TARGET_MINOR_VERSION
# So do not put a small version in upgrade-edge
echo "Error: OCP-25473 do not cover rollback, break the job"
return 1
if [[ "$INTERMEDIATE_MINOR_VERSION" == "$TARGET_MINOR_VERSION" ]]; then
# This is z stream retarget upgrade, e.g.: 4.y -> (4.y | 4.y+1) -> (4.y | 4.y+1)
first_upgrade_to_image=${intermediate_image}
second_upgrade_to_image=${TARGET}

if oc adm release info -o jsonpath='{.digest}' quay.io/openshift-release-dev/ocp-release:4.$((TARGET_MINOR_VERSION+1)).0-ec.0-x86_64 2>&1; then
# if third_upgrade_to_image not empty, we will retarget to it, but it will always be blocked
# this can make sure we always have z and y stream retarget in one job
latest_minor_version=$(( TARGET_MINOR_VERSION+1 ))
third_upgrade_to_image="$( oc adm release info -o jsonpath='{.digest}' quay.io/openshift-release-dev/ocp-release:4.${latest_minor_version}.0-ec.0-x86_64 )"
third_upgrade_to_image="quay.io/openshift-release-dev/ocp-release@${third_upgrade_to_image}"
fi

block_second=false
else
# If INTERMEDIATE_MINOR_VERSION < TARGET_MINOR_VERSION, then we will naver be able to upgrade to TARGET_MINOR_VERSION
# So do not put a small version in upgrade-edge
echo "Error: OCP-25473 do not cover rollback, break the job"
return 1
fi
fi
fi

local first_version second_version
first_version="$(oc adm release info -ojson "$intermediate_image" | jq -r '.metadata.version')"
first_version="$(oc adm release info -ojson "$first_upgrade_to_image" | jq -r '.metadata.version')"
FIRST_MINOR_VERSION="$(echo "$first_version" | cut -f2 -d.)"
second_version="$(oc adm release info -ojson "$intermediate_image" | jq -r '.metadata.version')"
second_version="$(oc adm release info -ojson "$second_upgrade_to_image" | jq -r '.metadata.version')"
SECOND_MINOR_VERSION="$(echo "$second_version" | cut -f2 -d.)"

# first_upgrade_to_image can be both nightly build or stable build
Expand Down