Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/azure-sdk-tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ jobs:
python eng/scripts/dispatch_checks.py --checks "$AZPYSDK_CHECKS" --wheel_dir $(pwd)/wheels azure-template
shell: bash
env:
TOX_PIP_IMPL: "uv"
AZPYSDK_PIP_IMPL: "uv"

- name: Install azure-sdk-tools on global pip env
run: |
Expand All @@ -118,4 +118,4 @@ jobs:
python eng/scripts/dispatch_checks.py --checks "$AZPYSDK_CHECKS" --wheel_dir $(pwd)/wheels azure-template
shell: bash
env:
TOX_PIP_IMPL: "pip"
AZPYSDK_PIP_IMPL: "pip"
2 changes: 1 addition & 1 deletion conda/conda_helper_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def _build_package_path_index() -> dict[str, str]:
This scans the sdk/ directory once and caches the result for all subsequent lookups.
"""
all_paths = glob.glob(os.path.join(SDK_DIR, "*", "*"))
# Exclude temp directories like .tox, .venv, __pycache__, etc.
# Exclude temp directories like .venv, __pycache__, etc.
return {
os.path.basename(p): p
for p in all_paths
Expand Down
6 changes: 1 addition & 5 deletions eng/pipelines/templates/jobs/ci.tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,7 @@ parameters:
- name: TestTimeoutInMinutes
type: number
default: 60
- name: ToxEnvParallel
type: string
default: 'all'
- name: UnsupportedToxEnvironments
- name: UnsupportedChecks
type: string
default: ''
- name: InjectedPackages
Expand Down Expand Up @@ -115,7 +112,6 @@ jobs:
CoverageArg: $(CoverageArg)
PythonVersion: $(PythonVersion)
CheckEnv: $(checks)
ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
InjectedPackages: $(InjectedPackages)
TestProxy: ${{ parameters.TestProxy }}
TestPipeline: ${{ parameters.TestPipeline }}
Expand Down
8 changes: 2 additions & 6 deletions eng/pipelines/templates/jobs/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ parameters:
- name: TestTimeoutInMinutes
type: number
default: 60
- name: ToxEnvParallel
type: string
default: 'all'
- name: InjectedPackages
type: string
default: ''
Expand All @@ -52,7 +49,7 @@ parameters:
- name: VerifyAutorest
type: boolean
default: false
- name: UnsupportedToxEnvironments
- name: UnsupportedChecks
type: string
default: ''
- name: TestProxy
Expand Down Expand Up @@ -324,9 +321,8 @@ jobs:
AfterTestSteps: ${{ parameters.AfterTestSteps }}
BuildTargetingString: ${{ parameters.BuildTargetingString }}
TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
InjectedPackages: ${{ parameters.InjectedPackages }}
UnsupportedToxEnvironments: ${{ parameters.UnsupportedToxEnvironments }}
UnsupportedChecks: ${{ parameters.UnsupportedChecks }}
TestProxy: ${{ parameters.TestProxy }}

- ${{ if ne(parameters.ServiceDirectory, 'auto') }}:
Expand Down
4 changes: 0 additions & 4 deletions eng/pipelines/templates/stages/archetype-sdk-client.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ parameters:
- name: TestTimeoutInMinutes
type: number
default: 60
- name: ToxEnvParallel
type: string
default: all
- name: InjectedPackages
type: string
default: ''
Expand Down Expand Up @@ -105,7 +102,6 @@ extends:
TestMarkArgument: ${{ parameters.TestMarkArgument }}
BuildTargetingString: ${{ parameters.BuildTargetingString }}
TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
ToxEnvParallel: ${{ parameters.ToxEnvParallel }}
InjectedPackages: ${{ parameters.InjectedPackages }}
BuildDocs: ${{ parameters.BuildDocs }}
DevFeedName: ${{ parameters.DevFeedName }}
Expand Down
7 changes: 3 additions & 4 deletions eng/pipelines/templates/steps/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ parameters:
CoverageArg: ''
CheckEnv: ""
RunCoverage: ne(variables['CoverageArg'], '--disablecov')
ToxEnvParallel: ''
InjectedPackages: ''
DevFeedName: 'public/azure-sdk-for-python'
TestProxy: false
Expand Down Expand Up @@ -56,7 +55,7 @@ steps:
inputs:
scriptPath: 'eng/scripts/set_checks.py'
arguments: >-
--unsupported="$(UnsupportedToxEnvironments)"
--unsupported="$(UnsupportedChecks)"
--override="$(ChecksOverride)"
--team-project="$(System.TeamProject)"

Expand All @@ -74,7 +73,7 @@ steps:
pwsh: true
ScriptType: InlineScript
Inline: >-
$env:TOX_PIP_IMPL="uv";
$env:AZPYSDK_PIP_IMPL="uv";
$account = (Get-AzContext).Account;
$env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id;
$env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants;
Expand All @@ -101,7 +100,7 @@ steps:

- ${{ else }}:
- pwsh: |
$env:TOX_PIP_IMPL="uv"
$env:AZPYSDK_PIP_IMPL="uv"
Write-Host (Get-Command python).Source

if ($env:TESTMARKARGUMENT) {
Expand Down
2 changes: 1 addition & 1 deletion eng/scripts/Language-Settings.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -520,7 +520,7 @@ function Get-python-DirectoriesForGeneration () {
| Get-ChildItem -Directory
| Where-Object { $_ -notmatch "-mgmt-" }
| Where-Object { (Test-Path "$_/tsp-location.yaml") }
# TODO: Reenable swagger generation when tox generate supports arbitrary generator versions
# TODO: Reenable swagger generation when generate supports arbitrary generator versions
# -or (Test-Path "$_/swagger/README.md")
}

Expand Down
File renamed without changes.
2 changes: 1 addition & 1 deletion eng/tools/azure-sdk-tools/azpysdk/dependency_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def _verify_installed_packages(self, executable: str, package_dir: str, staging_
)
return True

verify_script = os.path.join(REPO_ROOT, "eng/tox/verify_installed_packages.py")
verify_script = os.path.join(REPO_ROOT, "eng/scripts/verify_installed_packages.py")
verify_command = [verify_script, "--packages-file", packages_file]
verify_result = self.run_venv_command(executable, verify_command, cwd=package_dir)

Expand Down
4 changes: 2 additions & 2 deletions eng/tools/azure-sdk-tools/azpysdk/install_and_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def run(self, args: argparse.Namespace) -> int:

def check_coverage(self, executable: str, package_dir: str, package_name: str) -> int:
coverage_command = [
os.path.join(REPO_ROOT, "eng/tox/run_coverage.py"),
os.path.join(REPO_ROOT, "eng/scripts/run_coverage.py"),
"-t",
package_dir,
"-r",
Expand Down Expand Up @@ -132,7 +132,7 @@ def run_pytest(
"pytest exited with code 5 for %s, which is allowed for management or opt-out packages.",
package_name,
)
# Align with tox: skip coverage when tests are skipped entirely
# Skip coverage when tests are skipped entirely
return 0
else:
logger.error(f"pytest failed for {package_name} with exit code {pytest_result.returncode}.")
Expand Down
12 changes: 6 additions & 6 deletions eng/tools/azure-sdk-tools/azpysdk/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,11 +182,11 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
parser.print_help()
return 1

# default to uv if available, but respect an explicit TOX_PIP_IMPL setting
if "TOX_PIP_IMPL" not in os.environ:
# default to uv if available, but respect an explicit AZPYSDK_PIP_IMPL setting
if "AZPYSDK_PIP_IMPL" not in os.environ and "TOX_PIP_IMPL" not in os.environ:
uv_path = shutil.which("uv")
if uv_path:
os.environ["TOX_PIP_IMPL"] = "uv"
os.environ["AZPYSDK_PIP_IMPL"] = "uv"

# default to CFS feed unless --pypi is specified, but allow explicit env var override (e.g. for CI)
if args.pypi:
Expand All @@ -201,7 +201,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
os.environ["UV_DEFAULT_INDEX"] = CFS_INDEX_URL

# Log the feed being used
if os.environ.get("TOX_PIP_IMPL", None) == "uv":
if os.environ.get("AZPYSDK_PIP_IMPL", os.environ.get("TOX_PIP_IMPL")) == "uv":
logger.info("Installing from feed: %s", os.environ.get("UV_DEFAULT_INDEX"))
else:
logger.info("Installing from feed: %s", os.environ.get("PIP_INDEX_URL"))
Expand All @@ -215,9 +215,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
"--python requires --isolate to create a virtual environment with the specified Python version."
)

pip_impl = os.environ.get("TOX_PIP_IMPL", "pip").lower()
pip_impl = os.environ.get("AZPYSDK_PIP_IMPL", os.environ.get("TOX_PIP_IMPL", "pip")).lower()
if pip_impl != "uv":
parser.error("--python requires uv as the backend. Install uv or set TOX_PIP_IMPL=uv.")
parser.error("--python requires uv as the backend. Install uv or set AZPYSDK_PIP_IMPL=uv.")

try:
result = args.func(args)
Expand Down
13 changes: 6 additions & 7 deletions eng/tools/azure-sdk-tools/azpysdk/proxy_ports.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
"""Proxy port assignments for azpysdk checks.

This mapping mirrors the explicit `PROXY_URL` configuration found in
`eng/tox/tox.ini`. Because `dispatch_checks.py` runs multiple checks in
parallel, each check must bind to its own dedicated test-proxy port to avoid
races. Keeping this data in a single module allows both the CLI and the CI
launcher to share the same source of truth without having to parse the tox
configuration file at runtime.
This mapping defines the explicit ``PROXY_URL`` configuration for each check.
Because ``dispatch_checks.py`` runs multiple checks in parallel, each check
must bind to its own dedicated test-proxy port to avoid races. Keeping this
data in a single module allows both the CLI and the CI launcher to share the
same source of truth.
"""

from __future__ import annotations
Expand All @@ -16,7 +15,7 @@
DEFAULT_PROXY_URL = f"http://localhost:{DEFAULT_PROXY_PORT}"

# NOTE: `import_all` shares the same configuration as the legacy `depends`
# tox environment. All other entries match the tox environment names 1:1.
# check. All other entries match the check names 1:1.
CHECK_PROXY_PORTS: Dict[str, int] = {
"whl": DEFAULT_PROXY_PORT,
"sdist": 5001,
Expand Down
4 changes: 2 additions & 2 deletions eng/tools/azure-sdk-tools/ci_tools/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,7 @@ def get_package_from_repo_or_folder(req: str, prebuilt_wheel_dir: Optional[str]
attempts to find the package within the repo to install directly from path on disk.

During a CI build, it is preferred that the package is installed from a prebuilt wheel directory, as multiple CI environments attempting to install the relative
req can cause inconsistent installation issues during parallel tox environment execution.
req can cause inconsistent installation issues during parallel check execution.
"""

local_package = get_package_from_repo(req)
Expand All @@ -380,7 +380,7 @@ def get_package_from_repo_or_folder(req: str, prebuilt_wheel_dir: Optional[str]
if prebuilt_package:
# return the first package found, there should only be a single one matching given that our prebuilt wheel directory
# is populated by the replacement of dev_reqs.txt with the prebuilt wheels
# ref tox_harness replace_dev_reqs() calls
# ref replace_dev_reqs() calls
return os.path.join(prebuilt_wheel_dir, prebuilt_package[0])

if local_package:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
"""Utilities for resolving dependency sets for tox-style checks.
"""Utilities for resolving dependency sets for checks.

This module contains the logic previously hosted in ``eng/tox/install_depend_packages.py``
so that both the legacy tox entry point and the azpysdk checks can share a
single implementation.
so that the azpysdk checks can share a single implementation.
"""

import logging
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ def create_package_and_install(

def replace_dev_reqs(file: str, pkg_root: str, wheel_dir: Optional[str]) -> None:
"""Takes a target requirements file, replaces all local relative install locations with wheels assembled from whatever that target path was.
This is an extremely important step that runs on every dev_requirements.txt file before invoking any tox runs.
This is an extremely important step that runs on every dev_requirements.txt file before invoking any checks.

This is due to the fact that pip isn't multi-process-safe with the activity of installing a local relative requirement. .pyc files are updated
and removed in place, possibly causing a hang in the install process. When in_ci() is true, this function is run against every single requirement file.
Expand Down
11 changes: 6 additions & 5 deletions eng/tools/azure-sdk-tools/ci_tools/venv.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

This module centralizes all venv creation, pip install/uninstall, and
related utilities. The backend (``uv`` vs stdlib ``pip``) is chosen
via the ``TOX_PIP_IMPL`` environment variable:
via the ``AZPYSDK_PIP_IMPL`` environment variable (falls back to the
legacy ``TOX_PIP_IMPL`` for backward compatibility):

* ``"uv"`` → uses ``uv venv`` / ``uv pip``
* anything else (default ``"pip"``) → uses ``python -m venv`` / ``python -m pip``
Expand All @@ -24,10 +25,10 @@ def get_venv_call(python_exe: Optional[str] = None, python_version: Optional[str
:return: List of command arguments for venv.
:rtype: List[str]
"""
pip_impl = os.environ.get("TOX_PIP_IMPL", "pip").lower()
pip_impl = os.environ.get("AZPYSDK_PIP_IMPL", os.environ.get("TOX_PIP_IMPL", "pip")).lower()

if python_version and pip_impl != "uv":
raise ValueError("--python requires uv as the backend. Install uv or set TOX_PIP_IMPL=uv.")
raise ValueError("--python requires uv as the backend. Install uv or set AZPYSDK_PIP_IMPL=uv.")

# soon we will change this to default to uv
if pip_impl == "uv":
Expand All @@ -46,8 +47,8 @@ def get_pip_command(python_exe: Optional[str] = None) -> List[str]:
:return: List of command arguments for pip.
:rtype: List[str]
"""
# Check TOX_PIP_IMPL environment variable (aligns with tox.ini configuration)
pip_impl = os.environ.get("TOX_PIP_IMPL", "pip").lower()
# Check AZPYSDK_PIP_IMPL environment variable (falls back to legacy TOX_PIP_IMPL)
pip_impl = os.environ.get("AZPYSDK_PIP_IMPL", os.environ.get("TOX_PIP_IMPL", "pip")).lower()

# soon we will change this to default to uv
if pip_impl == "uv":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ def start_test_proxy(request) -> None:
root = os.getenv("BUILD_SOURCESDIRECTORY", repo_root)
_LOGGER.info("{} is calculated repo root".format(root))

# If we're in CI, allow for tox environment parallelization and write proxy output to a log file
# If we're in CI, allow for check parallelization and write proxy output to a log file
log = None
if in_ci():
log_suffix = _get_proxy_log_suffix()
Expand Down
28 changes: 16 additions & 12 deletions eng/tools/azure-sdk-tools/gh_tools/update_issue.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,36 +10,40 @@
import logging
from github import Github, Auth
from ci_tools.functions import discover_targeted_packages
from ci_tools.variables import discover_repo_root

logging.getLogger().setLevel(logging.INFO)
root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..", ".."))
root_dir = discover_repo_root()


def get_build_info(service_directory: str, package_name: str) -> str:
"""Get the build info from the build link."""
"""Get the pylint build info from the CI build logs."""
build_id = os.getenv("BUILD_BUILDID")
timeline_link = f"https://dev.azure.com/azure-sdk/internal/_apis/build/builds/{build_id}/timeline?api-version=6.0"

token = os.environ["SYSTEM_ACCESSTOKEN"]
AUTH_HEADERS = {"Authorization": f"Bearer {token}"}

try:
# Make the API request
response = requests.get(timeline_link, headers=AUTH_HEADERS)
response_json = json.loads(response.text)

for task in response_json["records"]:
if "Run Pylint Next" in task["name"]:
if "Run Pylint Next" in task.get("name", ""):
log_link = task["log"]["url"] + "?api-version=6.0"
# Get the log file from the build link
log_output = requests.get(log_link, headers=AUTH_HEADERS)
build_output = log_output.content.decode("utf-8")
new_output = (
build_output.split(
f"next-pylint: commands[3]> python /mnt/vss/_work/1/s/eng/tox/run_pylint.py -t {service_directory} --next=True"
)[1]
).split(f"ERROR:root:{package_name} exited with linting error")[0]
return new_output

# Extract next-pylint output from the dispatch_checks.py grouped log format
group_marker = f"{package_name} :: next-pylint ::"
if group_marker in build_output:
section = build_output.split(group_marker, 1)[1]
# The section ends at the next ##[endgroup] or the end of the log
if "##[endgroup]" in section:
section = section.split("##[endgroup]", 1)[0]
return section.strip()
Comment thread
JennyPng marked this conversation as resolved.

return "No next-pylint output found in build logs."
except Exception as e:
logging.error(f"Exception occurred while getting build info: {e}")
return "Error getting build info"
Expand Down Expand Up @@ -84,7 +88,7 @@ def main(targeted_packages):
description="""
This script is the single point for all checks invoked by CI within this repo. It works in two phases.
1. Identify which packages in the repo are in scope for this script invocation, based on a glob string and a service directory.
2. Invoke one or multiple `tox` environments for each package identified as in scope.
2. Invoke one or multiple checks for each package identified as in scope.

In the case of an environment invoking `pytest`, results can be collected in a junit xml file, and test markers can be selected via --mark_arg.
"""
Expand Down
Loading
Loading