Skip to content

DAOS-18778 test: Use ucx for Functional HW stages - only for test #18246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Nasf-Fan wants to merge 1 commit into
base: master
Choose a base branch
from
+52 −15
Draft

DAOS-18778 test: Use ucx for Functional HW stages - only for test #18246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,11 +370,14 @@ pipeline {
defaultValue: false,
description: 'Run the Functional Hardware Medium Verbs Provider test stage')
booleanParam(name: 'CI_medium_verbs_provider_md_on_ssd_TEST',
defaultValue: true,
defaultValue: false,
description: 'Run the Functional Hardware Medium Verbs Provider MD on SSD test stage')
booleanParam(name: 'CI_medium_ucx_provider_TEST',
defaultValue: false,
description: 'Run the Functional Hardware Medium UCX Provider test stage')
booleanParam(name: 'CI_medium_ucx_provider_md_on_ssd_TEST',
defaultValue: true,
description: 'Run the Functional Hardware Medium UCX Provider MD on SSD test stage')
booleanParam(name: 'CI_large_TEST',
defaultValue: false,
description: 'Run the Functional Hardware Large test stage')
Expand Down Expand Up @@ -1144,6 +1147,7 @@ pipeline {
pragma_suffix: '-hw-medium',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL,
next_version: next_version(),
other_packages: 'mercury-libfabric mercury-ucx',
stage_tags: 'hw,medium,-provider',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
nvme: 'auto',
Expand All @@ -1157,6 +1161,7 @@ pipeline {
pragma_suffix: '-hw-medium-md-on-ssd',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL,
next_version: next_version(),
other_packages: 'mercury-libfabric mercury-ucx',
stage_tags: 'hw,medium,-provider',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
nvme: 'auto_md_on_ssd',
Expand All @@ -1170,6 +1175,7 @@ pipeline {
pragma_suffix: '-hw-medium-vmd',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_VMD_LABEL,
next_version: next_version(),
other_packages: 'mercury-libfabric mercury-ucx',
stage_tags: 'hw_vmd,medium',
/* groovylint-disable-next-line UnnecessaryGetter */
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
Expand All @@ -1184,6 +1190,7 @@ pipeline {
pragma_suffix: '-hw-medium-verbs-provider',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL,
next_version: next_version(),
other_packages: 'mercury-libfabric mercury-ucx',
stage_tags: 'hw,medium,provider',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
default_nvme: 'auto',
Expand All @@ -1198,11 +1205,12 @@ pipeline {
pragma_suffix: '-hw-medium-verbs-provider-md-on-ssd',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL,
next_version: next_version(),
other_packages: 'mercury-libfabric mercury-ucx',
stage_tags: 'hw,medium,provider',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
default_nvme: 'auto_md_on_ssd',
provider: 'ofi+verbs;ofi_rxm',
run_if_pr: true,
run_if_pr: false,
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
Expand All @@ -1212,6 +1220,7 @@ pipeline {
pragma_suffix: '-hw-medium-ucx-provider',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_UCX_PROVIDER_LABEL,
next_version: next_version(),
other_packages: 'mercury-ucx',
stage_tags: 'hw,medium,provider',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
default_nvme: 'auto',
Expand All @@ -1221,11 +1230,27 @@ pipeline {
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Medium UCX Provider MD on SSD': getFunctionalTestStage(
name: 'Functional Hardware Medium UCX Provider MD on SSD',
pragma_suffix: '-hw-medium-ucx-provider-md-on-ssd',
label: params.FUNCTIONAL_HARDWARE_MEDIUM_UCX_PROVIDER_LABEL,
next_version: next_version(),
other_packages: 'mercury-ucx',
stage_tags: 'hw,medium,provider',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
default_nvme: 'auto_md_on_ssd',
provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'),
run_if_pr: true,
run_if_landing: false,
job_status: job_status_internal,
image_version: 'el9.7'
),
'Functional Hardware Large': getFunctionalTestStage(
name: 'Functional Hardware Large',
pragma_suffix: '-hw-large',
label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL,
next_version: next_version(),
other_packages: 'mercury-libfabric mercury-ucx',
stage_tags: 'hw,large',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
default_nvme: 'auto',
Expand All @@ -1239,6 +1264,7 @@ pipeline {
pragma_suffix: '-hw-large-md-on-ssd',
label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL,
next_version: next_version(),
other_packages: 'mercury-libfabric mercury-ucx',
stage_tags: 'hw,large',
default_tags: startedByTimer() ? 'pr daily_regression' : 'pr',
default_nvme: 'auto_md_on_ssd',
Expand Down
37 changes: 24 additions & 13 deletions src/tests/ftest/util/network_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""
(C) Copyright 2022-2024 Intel Corporation.
(C) Copyright 2025 Hewlett Packard Enterprise Development LP
(C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP

SPDX-License-Identifier: BSD-2-Clause-Patent
"""
Expand All @@ -16,9 +16,9 @@
# Order here is used to select default provider in environment_utils
SUPPORTED_PROVIDERS = (
"ofi+cxi",
"ofi+verbs;ofi_rxm",
"ucx+dc_x",
"ucx+ud_x",
"ucx+dc_x",
"ofi+verbs;ofi_rxm",
"ofi+tcp",
"ofi+tcp;ofi_rxm",
"ofi+opx"
Expand Down Expand Up @@ -253,10 +253,8 @@
Args:
logger (Logger): logger for the messages produced by this method
hosts (NodeSet): hosts from which to gather the information
filter_provider (list, optional): list of supported providers to filter by.
Defaults to None.
filter_device (list, optional): list of supported devices to filter by.
Defaults to None.
filter_provider (list, optional): list of providers to be included. Defaults to None.
filter_device (list, optional): list of devices to be included. Defaults to None.
verbose (bool, optional): display command details. Defaults to True.

Returns:
Expand All @@ -266,7 +264,7 @@
command = "hg_info"
result = run_remote(logger, hosts, command, verbose=verbose, stderr=True)
providers = {}
if result.passed:

Check warning on line 267 in src/tests/ftest/util/network_utils.py

View workflow job for this annotation

GitHub Actions / Pylint check 

too-many-nested-blocks, Too many nested blocks (6/5)
# Find all supported providers
for data in result.output:
if not data.stdout:
Expand All @@ -281,14 +279,27 @@
class_protocol_device = re.findall(
r'(\S+) +([\S]+) +([\S]+)$', without_header, re.MULTILINE)
for _class, protocol, device in class_protocol_device:
if ":" in device:
# e.g. convert mlx5_0:1 => mlx5_0
device = device.split(":")[0]
if filter_device and device not in filter_device:
continue
provider = f"{_class}+{protocol}"
if filter_provider and provider not in filter_provider:
continue
if device not in device_providers:
device_providers[device] = set()
device_providers[device].add(provider)
_class_protocols = []
if "_" in protocol:
_protocol = protocol.split("_")
if _protocol[0] in ("dc", "rc", "ud") and _protocol[1] in ("mlx5"):
# Add ucx synonyms, e.g. convert ud_mlx5 => ud_mlx5, ud_x, ud
for synonym in (f"_{_protocol[1]}", "_x", ""):
_class_protocols.append(f"{_class}+{_protocol[0]}{synonym}")
else:
_class_protocols.append(f"{_class}+{protocol}")

for provider in _class_protocols:
if filter_provider and provider not in filter_provider:
continue
if device not in device_providers:
device_providers[device] = set()
device_providers[device].add(provider)

for device, provider_set in device_providers.items():
if device not in providers:
Expand Down
Loading