From adef48a49f6593413317af46ef5d86d0338921d1 Mon Sep 17 00:00:00 2001 From: aryasadeghi1 Date: Mon, 28 Sep 2026 14:02:57 +1000 Subject: [PATCH 1/3] build: migrate CLI packages to pyproject.toml Preserve package payloads and dependencies while updating build discovery, version checks, and release consumers. Pin CI to azdev 0.2.14 and retain only justified setuptools compatibility constraints. Related to Azure/azure-cli-extensions#7740. --- .../templates/automation_test.yml | 8 + .azure-pipelines/templates/azdev_setup.yml | 16 +- .github/CODEOWNERS | 4 +- .github/actions/env-setup/action.yml | 2 +- azure-cli.pyproj | 6 +- azure-cli2017.pyproj | 9 +- azure-pipelines.yml | 26 +- requirements.txt | 6 +- scripts/ci/build.sh | 10 +- scripts/ci/check_package_versions.py | 32 ++ scripts/ci/version.sh | 8 +- scripts/install_full.sh | 2 +- scripts/regression_test/regression_test.yml | 4 +- scripts/release/debian/build.sh | 6 +- .../release/homebrew/docker/requirements.txt | 8 +- scripts/release/homebrew/docker/run.sh | 2 +- scripts/release/pypi/build.sh | 17 +- .../release/rpm/test_azurelinux_in_docker.sh | 7 +- scripts/release/rpm/test_rpm_in_docker.sh | 7 +- src/azure-cli-core/pyproject.toml | 66 ++++ src/azure-cli-core/setup.py | 92 ----- src/azure-cli-telemetry/pyproject.toml | 39 +++ src/azure-cli-telemetry/setup.py | 51 --- src/azure-cli-testsdk/pyproject.toml | 40 +++ src/azure-cli-testsdk/setup.py | 54 --- src/azure-cli/pyproject.toml | 185 ++++++++++ src/azure-cli/requirements.py3.Darwin.txt | 2 +- src/azure-cli/requirements.py3.Linux.txt | 2 +- src/azure-cli/requirements.py3.windows.txt | 2 +- src/azure-cli/setup.py | 223 ------------ .../tests/test_packaging_migration.py | 321 ++++++++++++++++++ .../tests/verify_package_versions.py | 6 +- .../automation/tests/verify_readme_history.py | 42 ++- tools/automation/utilities/packaging.py | 92 +++++ tools/automation/utilities/path.py | 18 +- tools/automation/verify/default_modules.py | 8 +- tools/setup.py | 3 + 37 files changed, 928 insertions(+), 498 deletions(-) create mode 100644 scripts/ci/check_package_versions.py create mode 100644 src/azure-cli-core/pyproject.toml delete mode 100644 src/azure-cli-core/setup.py create mode 100644 src/azure-cli-telemetry/pyproject.toml delete mode 100755 src/azure-cli-telemetry/setup.py create mode 100644 src/azure-cli-testsdk/pyproject.toml delete mode 100644 src/azure-cli-testsdk/setup.py create mode 100644 src/azure-cli/pyproject.toml delete mode 100644 src/azure-cli/setup.py create mode 100644 tools/automation/tests/test_packaging_migration.py create mode 100644 tools/automation/utilities/packaging.py diff --git a/.azure-pipelines/templates/automation_test.yml b/.azure-pipelines/templates/automation_test.yml index dab01694208..06782c2f330 100644 --- a/.azure-pipelines/templates/automation_test.yml +++ b/.azure-pipelines/templates/automation_test.yml @@ -27,6 +27,12 @@ parameters: displayName: Job name type: string default: '' +- name: AzdevPackage + type: string + default: 'azdev==0.2.14' +- name: DiffToolPackage + type: string + default: '' steps: - task: UsePythonVersion@0 @@ -36,6 +42,8 @@ steps: - template: ./azdev_setup.yml parameters: EnableCompactAAZ: true + AzdevPackage: ${{ parameters.AzdevPackage }} + DiffToolPackage: ${{ parameters.DiffToolPackage }} - bash: | set -ev diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 95c81836dee..e07b3bfcd73 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -8,6 +8,13 @@ parameters: - name: EnableCompactAAZ type: boolean default: false + # Override with a candidate wheel path when validating before the azdev release. + - name: AzdevPackage + type: string + default: 'azdev==0.2.14' + - name: DiffToolPackage + type: string + default: '' steps: - bash: | set -ev @@ -16,8 +23,11 @@ steps: chmod +x env/bin/activate . env/bin/activate - python -m pip install -U pip "setuptools<81" - pip install --upgrade "azdev==0.2.13" + python -m pip install -U pip "setuptools>=78.1.1" + if [ -n "$DIFF_TOOL_PACKAGE" ]; then + pip install --upgrade "$DIFF_TOOL_PACKAGE" + fi + pip install --upgrade "$AZDEV_PACKAGE" azdev --version if [ -z "$CLI_EXT_REPO_PATH" ]; then @@ -37,5 +47,7 @@ steps: az --version displayName: 'azdev setup' env: + AZDEV_PACKAGE: ${{ parameters.AzdevPackage }} + DIFF_TOOL_PACKAGE: ${{ parameters.DiffToolPackage }} CLI_REPO_PATH: ${{ parameters.CLIRepoPath }} CLI_EXT_REPO_PATH: ${{ parameters.CLIExtensionRepoPath }} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12e6d10f0a0..d727b2c523a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,8 +22,8 @@ /src/azure-cli/ @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad -/src/azure-cli-core/setup.py @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad -/src/azure-cli/setup.py @Azure/act-identity-squad @Azure/act-codegen-extensibility-squad +/src/azure-cli-core/pyproject.toml @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad +/src/azure-cli/pyproject.toml @Azure/act-identity-squad @Azure/act-codegen-extensibility-squad /src/azure-cli/requirements.*.txt @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad /src/azure-cli/azure/cli/command_modules/acr/ @Azure/act-observability-squad diff --git a/.github/actions/env-setup/action.yml b/.github/actions/env-setup/action.yml index 19d40dd7084..a4beb97742f 100644 --- a/.github/actions/env-setup/action.yml +++ b/.github/actions/env-setup/action.yml @@ -29,7 +29,7 @@ runs: python -m venv env chmod +x env/bin/activate source ./env/bin/activate - pip install azdev + pip install "azdev==0.2.14" azdev --version cd ../ azdev setup -c azure-cli --debug diff --git a/azure-cli.pyproj b/azure-cli.pyproj index 687601ef6dd..dd3ac6195a0 100644 --- a/azure-cli.pyproj +++ b/azure-cli.pyproj @@ -75,7 +75,7 @@ - + @@ -84,11 +84,11 @@ - + - + diff --git a/azure-cli2017.pyproj b/azure-cli2017.pyproj index dc1d455ca7d..ecb6467b380 100644 --- a/azure-cli2017.pyproj +++ b/azure-cli2017.pyproj @@ -87,7 +87,7 @@ - + @@ -118,8 +118,7 @@ - - + @@ -129,7 +128,7 @@ - + @@ -963,7 +962,7 @@ - + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f0626e557e8..2cdf1ec49c0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -541,6 +541,28 @@ jobs: pythonVersion: '$(python.version)' module: 'azure-cli-core' +- job: TestPackagingMetadata + displayName: Test Packaging Metadata + timeoutInMinutes: 10 + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python310: + python.version: '3.10' + Python314: + python.version: '3.14' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + - bash: | + set -e + python -m pip install docutils requests packaging tomli + python -B -m unittest discover -s tools/automation/tests -t tools -p test_packaging_migration.py -v + python -B scripts/ci/check_package_versions.py + displayName: Validate source metadata and packaging automation + - job: TestTelemetry displayName: Unit Test for Telemetry timeoutInMinutes: 10 @@ -577,7 +599,7 @@ jobs: displayName: 'Use Python $(python.version)' inputs: versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel build "setuptools<81" + - bash: pip install --upgrade pip wheel build "setuptools>=78.1.1" displayName: 'Install pip and wheel' - bash: ./scripts/ci/test_profile_integration.sh displayName: 'Run Integration Test against Profiles' @@ -598,7 +620,7 @@ jobs: displayName: 'Use Python $(python.version)' inputs: versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel build "setuptools<81" + - bash: pip install --upgrade pip wheel build "setuptools>=78.1.1" displayName: 'Install pip and wheel setuptools' - bash: ./scripts/ci/test_extensions.sh displayName: 'Load extensions' diff --git a/requirements.txt b/requirements.txt index d854618ebf9..f658b6490fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ # basic -# Cap setuptools<81: newer setuptools breaks the CLI's setup.py-based builds -# (81 removes setup.py --dry-run and changes distutils command signatures; -# 82 removes pkg_resources). Pin to the last 80.x line. -setuptools>=65.5.1,<81 +# Security floor for setuptools; CLI packages use the PEP 517 build backend. +setuptools>=78.1.1 pip>=9.0.1 diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index c5a49c759f5..845a5d8b67b 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -55,13 +55,15 @@ title 'Build Azure CLI and its command modules' # provisioning the frontend is easy to miss. Install it here if it is absent rather # than failing partway through the build. python -c 'import build' 2>/dev/null || python -m pip install --disable-pip-version-check -q build +python -c 'import tomllib' 2>/dev/null || python -m pip install --disable-pip-version-check -q 'tomli>=2.0.1' +python "$script_dir/check_package_versions.py" -for setup_file in $(find src -name 'setup.py'); do - pushd $(dirname ${setup_file}) >/dev/null +for pyproject_file in $(find src -name 'pyproject.toml'); do + pushd $(dirname ${pyproject_file}) >/dev/null echo "Building module at $(pwd) ..." # --no-isolation builds against the environment prepared by the caller rather than - # provisioning a fresh one, which keeps the caller's setuptools pin in force and - # avoids requiring outbound network access inside the packaging test containers. + # provisioning a fresh one. The caller must provide the build requirements; + # this avoids fetching them inside the packaging test containers. python -m build --wheel --no-isolation --outdir $output_dir python -m build --sdist --no-isolation --outdir $sdist_dir popd >/dev/null diff --git a/scripts/ci/check_package_versions.py b/scripts/ci/check_package_versions.py new file mode 100644 index 00000000000..062011d8f5d --- /dev/null +++ b/scripts/ci/check_package_versions.py @@ -0,0 +1,32 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Validate source versions before building release or nightly distributions.""" + +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[2] +PACKAGES = ('azure-cli', 'azure-cli-core', 'azure-cli-telemetry', 'azure-cli-testsdk') +sys.path.insert(0, str(ROOT / 'tools')) + +from automation.utilities.packaging import get_package_version, tomllib # noqa: E402 + + +def main(): + versions = {} + for name in PACKAGES: + versions[name] = get_package_version(ROOT / 'src' / name) + with (ROOT / 'src/azure-cli/pyproject.toml').open('rb') as stream: + dependencies = tomllib.load(stream)['project']['dependencies'] + expected = 'azure-cli-core==' + versions['azure-cli-core'] + if expected not in dependencies or versions['azure-cli'] != versions['azure-cli-core']: + raise ValueError('CLI and core versions must match, including the dependency: ' + expected) + for name, version in versions.items(): + print('{}: {}'.format(name, version)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/ci/version.sh b/scripts/ci/version.sh index 4494e89ac9c..907ac53458c 100755 --- a/scripts/ci/version.sh +++ b/scripts/ci/version.sh @@ -32,11 +32,13 @@ for each in $(find src -name __init__.py); do fi done -for each in $(find src -name setup.py); do +for each in $(find src -name pyproject.toml); do if [ "$platform" == "Darwin" ]; then - sed -i "" "s/^VERSION = [\"']\(.*\)[\"']/VERSION = \"\1.$version\"/" $each + sed -i "" "s/^version = [\"']\(.*\)[\"']/version = \"\1.$version\"/" "$each" + sed -i "" "s/azure-cli-core==\([^\"']*\)/azure-cli-core==\1.$version/" "$each" else - sed -i "s/^VERSION = [\"']\(.*\)[\"']/VERSION = \"\1.$version\"/" $each + sed -i "s/^version = [\"']\(.*\)[\"']/version = \"\1.$version\"/" "$each" + sed -i "s/azure-cli-core==\([^\"']*\)/azure-cli-core==\1.$version/" "$each" fi done diff --git a/scripts/install_full.sh b/scripts/install_full.sh index bdda1836ef8..62e3dae3aa2 100755 --- a/scripts/install_full.sh +++ b/scripts/install_full.sh @@ -16,7 +16,7 @@ REPO_ROOT="$(dirname ${BASH_SOURCE[0]})/.." pushd ${REPO_ROOT} > /dev/null -find src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps +find src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps pip install -r ./src/azure-cli/requirements.$(python ./scripts/get-python-version.py).$(uname).txt popd > /dev/null diff --git a/scripts/regression_test/regression_test.yml b/scripts/regression_test/regression_test.yml index f618b0c0dfb..7149cb978cd 100644 --- a/scripts/regression_test/regression_test.yml +++ b/scripts/regression_test/regression_test.yml @@ -46,8 +46,8 @@ jobs: echo "'PACKAGE' and 'TARGET_PACKAGE_VERSION' are required" exit 1 fi - sed -i "s/'$(PACKAGE)==.*'/'$(PACKAGE)==$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py - sed -i "s/'$(PACKAGE)~=.*'/'$(PACKAGE)~=$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py + sed -i "s/'$(PACKAGE)==.*'/'$(PACKAGE)==$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/pyproject.toml + sed -i "s/'$(PACKAGE)~=.*'/'$(PACKAGE)~=$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/pyproject.toml sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.windows.txt sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Linux.txt sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Darwin.txt diff --git a/scripts/release/debian/build.sh b/scripts/release/debian/build.sh index 5846a7dc2b8..580ac5c1817 100755 --- a/scripts/release/debian/build.sh +++ b/scripts/release/debian/build.sh @@ -37,12 +37,12 @@ $PYTHON_SRC_DIR/*/configure --srcdir $PYTHON_SRC_DIR/* --prefix $WORKDIR/python_ make make install -# Cap setuptools<81: 81 removes setup.py --dry-run and changes distutils command signatures (82 removes pkg_resources); the CLI build relies on setup.py. -$WORKDIR/python_env/bin/python3 -m pip install --upgrade pip "setuptools<81" +# Keep setuptools at or above the security floor. +$WORKDIR/python_env/bin/python3 -m pip install --upgrade pip "setuptools>=78.1.1" export PATH=$PATH:$WORKDIR/python_env/bin -find ${WORKDIR}/src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip3 install --no-deps +find ${WORKDIR}/src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip3 install --no-deps pip3 install -r ${WORKDIR}/src/azure-cli/requirements.py3.$(uname).txt $WORKDIR/python_env/bin/python3 ${WORKDIR}/scripts/trim_sdk.py diff --git a/scripts/release/homebrew/docker/requirements.txt b/scripts/release/homebrew/docker/requirements.txt index 0e37f95638a..0dd9c04cdc4 100644 --- a/scripts/release/homebrew/docker/requirements.txt +++ b/scripts/release/homebrew/docker/requirements.txt @@ -1,7 +1,7 @@ -# Pin setuptools<81 to stay aligned with the CLI-wide cap: homebrew-pypi-poet -# (abandoned since 2018) uses pkg_resources, which was removed in setuptools 82+, -# and 81 changes setup.py/distutils behavior. See https://github.com/Azure/azure-cli/issues/32800 -setuptools<81 +# homebrew-pypi-poet uses pkg_resources, removed in setuptools 82. +# Keep this cap local to formula-generation tooling, with the security floor. +# See https://github.com/Azure/azure-cli/issues/32800 +setuptools>=78.1.1,<82 homebrew-pypi-poet~=0.10.0 jinja2~=3.1.6 requests>=2.20.0 diff --git a/scripts/release/homebrew/docker/run.sh b/scripts/release/homebrew/docker/run.sh index 1ab98f2da92..b19e7a42bd3 100755 --- a/scripts/release/homebrew/docker/run.sh +++ b/scripts/release/homebrew/docker/run.sh @@ -13,7 +13,7 @@ fi pip install wheel pip install -U pip pip install -r $root/requirements.txt -find /mnt/src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps +find /mnt/src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps pip install -r /mnt/src/azure-cli/requirements.py3.Darwin.txt pip list diff --git a/scripts/release/pypi/build.sh b/scripts/release/pypi/build.sh index 6857dbc9b70..96535056e63 100755 --- a/scripts/release/pypi/build.sh +++ b/scripts/release/pypi/build.sh @@ -14,14 +14,12 @@ cd $BUILD_SOURCESDIRECTORY branch=$1 echo "Branch $branch" -echo "Search setup files from `pwd`." +echo "Search pyproject.toml files from `pwd`." python --version -# Cap setuptools<81: setuptools 82 removes pkg_resources, which azure-cli still declares as a -# runtime dependency. The builds below go through the PEP 517 frontend, but --no-isolation means -# they use this setuptools too. -# `build` is that frontend. -pip install -U pip "setuptools<81" wheel build +# Provision the PEP 517 frontend and backend for --no-isolation builds, +# keeping setuptools at or above the security floor. +pip install -U pip "setuptools>=78.1.1" wheel build pip list script_dir=`cd $(dirname $BASH_SOURCE[0]); pwd` @@ -30,8 +28,11 @@ if [[ ! $branch =~ ^release ]]; then . $script_dir/../../ci/version.sh post`date -u '+%Y%m%d%H%M%S'` fi -for setup_file in $(find src -name 'setup.py' | grep -v azure-cli-testsdk); do - pushd `dirname $setup_file` +python -c 'import tomllib' 2>/dev/null || python -m pip install --disable-pip-version-check -q 'tomli>=2.0.1' +python "$script_dir/../../ci/check_package_versions.py" + +for pyproject_file in $(find src -name 'pyproject.toml' | grep -v azure-cli-testsdk); do + pushd `dirname $pyproject_file` python -m build --wheel --no-isolation --outdir $BUILD_STAGINGDIRECTORY python -m build --sdist --no-isolation --outdir $BUILD_STAGINGDIRECTORY popd diff --git a/scripts/release/rpm/test_azurelinux_in_docker.sh b/scripts/release/rpm/test_azurelinux_in_docker.sh index 35f74c72d60..ae4b2794b48 100644 --- a/scripts/release/rpm/test_azurelinux_in_docker.sh +++ b/scripts/release/rpm/test_azurelinux_in_docker.sh @@ -14,10 +14,9 @@ time az self-test time az --version cd /azure-cli/ -# Cap setuptools<81: 81 removes setup.py --dry-run and changes distutils command signatures (82 removes pkg_resources). -# scripts/ci/build.sh builds with `python -m build --no-isolation`, so this pin is the setuptools the build uses. -# `build` is the PEP 517 frontend that script invokes. -python -m pip install --upgrade "setuptools<81" build +# Provision build and setuptools for scripts/ci/build.sh's --no-isolation builds, +# keeping setuptools at or above the security floor. +python -m pip install --upgrade "setuptools>=78.1.1" build ./scripts/ci/build.sh # From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh index de1be306174..8eb556eba1e 100644 --- a/scripts/release/rpm/test_rpm_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -16,10 +16,9 @@ time az self-test time az --version cd /azure-cli/ -# Cap setuptools<81: 81 removes setup.py --dry-run and changes distutils command signatures (82 removes pkg_resources). -# scripts/ci/build.sh builds with `python -m build --no-isolation`, so this pin is the setuptools the build uses. -# `build` is the PEP 517 frontend that script invokes. -python -m pip install --upgrade pip "setuptools<81" build +# Provision build and setuptools for scripts/ci/build.sh's --no-isolation builds, +# keeping setuptools at or above the security floor. +python -m pip install --upgrade pip "setuptools>=78.1.1" build ./scripts/ci/build.sh # From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. diff --git a/src/azure-cli-core/pyproject.toml b/src/azure-cli-core/pyproject.toml new file mode 100644 index 00000000000..25add95ce33 --- /dev/null +++ b/src/azure-cli-core/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli-core" +version = "2.90.0" +description = "Microsoft Azure Command-Line Tools Core Module" +readme = { file = "README.rst", content-type = "text/x-rst" } +requires-python = ">=3.10.0" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + 'argcomplete~=3.5.2', + 'azure-cli-telemetry==1.1.0.*', + 'azure-core~=1.39.0', + 'azure-mgmt-core>=1.2.0,<2', + 'cryptography', + # On Linux, the distribution (Ubuntu, Debian, etc) and version are logged in telemetry + 'distro; sys_platform == "linux"', + 'humanfriendly~=10.0', + 'jmespath', + 'knack~=0.14.0', + 'microsoft-security-utilities-secret-masker~=1.0.0b4', + 'msal-extensions==1.3.1', + 'msal[broker]==1.36.0; sys_platform == "win32"', + 'msal==1.36.0; sys_platform != "win32"', + 'packaging>=20.9', + # pkginfo>=1.12.0 reads the spec-defined wheel METADATA / unpacked .dist-info + # layout produced by modern wheel/setuptools (no metadata.json). Required so + # WheelExtension.get_metadata works without the legacy wheel==0.30.0 artifact. + 'pkginfo>=1.12.0', + # psutil can't install on cygwin: https://github.com/Azure/azure-cli/issues/9399 + 'psutil>=5.9; sys_platform != "cygwin"', + 'PyJWT>=2.1.0', + 'pyopenssl>=17.1.0', # https://github.com/pyca/pyopenssl/pull/612 + 'py-deviceid', + 'requests[socks]', +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false + +[tool.setuptools.packages.find] +namespaces = false +exclude = ["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"] + +[tool.setuptools.package-data] +"azure.cli.core" = ["auth/landing_pages/*.html", "commandIndex.latest.json", "helpIndex.latest.json"] \ No newline at end of file diff --git a/src/azure-cli-core/setup.py b/src/azure-cli-core/setup.py deleted file mode 100644 index bc0c5828e34..00000000000 --- a/src/azure-cli-core/setup.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup, find_packages - -VERSION = "2.90.0" - -# If we have source, validate that our version numbers match -# This should prevent uploading releases with mismatched versions. -try: - with open('azure/cli/core/__init__.py', 'r', encoding='utf-8') as f: - content = f.read() -except OSError: - pass -else: - import re - import sys - - m = re.search(r'__version__\s*=\s*[\'"](.+?)[\'"]', content) - if not m: - print('Could not find __version__ in azure/cli/core/__init__.py') - sys.exit(1) - if m.group(1) != VERSION: - print('Expected __version__ = "{}"; found "{}"'.format(VERSION, m.group(1))) - sys.exit(1) - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', -] - -DEPENDENCIES = [ - 'argcomplete~=3.5.2', - 'azure-cli-telemetry==1.1.0.*', - 'azure-core~=1.39.0', - 'azure-mgmt-core>=1.2.0,<2', - 'cryptography', - # On Linux, the distribution (Ubuntu, Debian, etc) and version are logged in telemetry - 'distro; sys_platform == "linux"', - 'humanfriendly~=10.0', - 'jmespath', - 'knack~=0.14.0', - 'microsoft-security-utilities-secret-masker~=1.0.0b4', - 'msal-extensions==1.3.1', - 'msal[broker]==1.36.0; sys_platform == "win32"', - 'msal==1.36.0; sys_platform != "win32"', - 'packaging>=20.9', - # pkginfo>=1.12.0 reads the spec-defined wheel METADATA / unpacked .dist-info - # layout produced by modern wheel/setuptools (no metadata.json). Required so - # WheelExtension.get_metadata works without the legacy wheel==0.30.0 artifact. - 'pkginfo>=1.12.0', - # psutil can't install on cygwin: https://github.com/Azure/azure-cli/issues/9399 - 'psutil>=5.9; sys_platform != "cygwin"', - 'PyJWT>=2.1.0', - 'pyopenssl>=17.1.0', # https://github.com/pyca/pyopenssl/pull/612 - 'py-deviceid', - 'requests[socks]', -] - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() - -setup( - name='azure-cli-core', - version=VERSION, - description='Microsoft Azure Command-Line Tools Core Module', - long_description=README, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"]), - install_requires=DEPENDENCIES, - python_requires='>=3.10.0', - package_data={'azure.cli.core': ['auth/landing_pages/*.html', 'commandIndex.latest.json', 'helpIndex.latest.json']} -) diff --git a/src/azure-cli-telemetry/pyproject.toml b/src/azure-cli-telemetry/pyproject.toml new file mode 100644 index 00000000000..9998646086e --- /dev/null +++ b/src/azure-cli-telemetry/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli-telemetry" +version = "1.1.0" +description = "Microsoft Azure CLI Telemetry Package" +dynamic = ["readme"] +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + 'applicationinsights>=0.11.1,<0.12', + 'portalocker>=1.6,<4', +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false +packages = ["azure.cli.telemetry", "azure.cli.telemetry.components"] + +[tool.setuptools.dynamic] +readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" } \ No newline at end of file diff --git a/src/azure-cli-telemetry/setup.py b/src/azure-cli-telemetry/setup.py deleted file mode 100755 index 63a6b21cc61..00000000000 --- a/src/azure-cli-telemetry/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup - -VERSION = "1.1.0" - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', -] - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() -with open('HISTORY.rst', 'r', encoding='utf-8') as f: - HISTORY = f.read() - -setup( - name='azure-cli-telemetry', - version=VERSION, - description='Microsoft Azure CLI Telemetry Package', - long_description=README + '\n\n' + HISTORY, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - install_requires=[ - 'applicationinsights>=0.11.1,<0.12', - 'portalocker>=1.6,<4', - ], - packages=[ - 'azure.cli.telemetry', - 'azure.cli.telemetry.components' - ] -) diff --git a/src/azure-cli-testsdk/pyproject.toml b/src/azure-cli-testsdk/pyproject.toml new file mode 100644 index 00000000000..efb4889660f --- /dev/null +++ b/src/azure-cli-testsdk/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli-testsdk" +version = "0.3.0" +description = "Microsoft Azure Command-Line Tools Test SDK Module" +dynamic = ["readme"] +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + 'ConfigArgParse>=0.12.0', + 'jmespath', + 'vcrpy>=1.10.3', + 'pytest', +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false +packages = ["azure.cli.testsdk", "azure.cli.testsdk.scenario_tests"] + +[tool.setuptools.dynamic] +readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" } \ No newline at end of file diff --git a/src/azure-cli-testsdk/setup.py b/src/azure-cli-testsdk/setup.py deleted file mode 100644 index d32fe0bebcc..00000000000 --- a/src/azure-cli-testsdk/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup - -VERSION = "0.3.0" - -CLASSIFIERS = [ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', -] - -DEPENDENCIES = [ - 'ConfigArgParse>=0.12.0', - 'jmespath', - 'vcrpy>=1.10.3', - 'pytest' -] - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() -with open('HISTORY.rst', 'r', encoding='utf-8') as f: - HISTORY = f.read() - -setup( - name='azure-cli-testsdk', - version=VERSION, - description='Microsoft Azure Command-Line Tools Test SDK Module', - long_description=README + '\n\n' + HISTORY, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - packages=[ - 'azure.cli.testsdk', - 'azure.cli.testsdk.scenario_tests', - ], - install_requires=DEPENDENCIES -) diff --git a/src/azure-cli/pyproject.toml b/src/azure-cli/pyproject.toml new file mode 100644 index 00000000000..08d97fc8bff --- /dev/null +++ b/src/azure-cli/pyproject.toml @@ -0,0 +1,185 @@ +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli" +version = "2.90.0" +description = "Microsoft Azure Command-Line Tools" +readme = { file = "README.rst", content-type = "text/x-rst" } +requires-python = ">=3.10.0" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "antlr4-python3-runtime~=4.13.1", + 'azure-appconfiguration==1.10.0b1', + 'azure-ai-projects~=1.0.0', + 'azure-batch==15.0.0b1', + 'azure-cli-core==2.90.0', + 'azure-cosmos~=3.0,>=3.0.2', + 'azure-data-tables==12.4.0', + 'azure-datalake-store~=1.0.1', + 'azure-keyvault-administration==4.8.0b2', + 'azure-keyvault-certificates==4.7.0', + 'azure-keyvault-keys==4.12.0b2', + 'azure-keyvault-secrets==4.7.0', + 'azure-keyvault-securitydomain==1.0.0b1', + 'azure-mgmt-advisor==9.0.0', + 'azure-mgmt-apimanagement==4.0.0', + 'azure-mgmt-appconfiguration==6.0.0b2', + 'azure-mgmt-appcontainers==2.0.0', + 'azure-mgmt-applicationinsights~=1.0.0', + 'azure-mgmt-authorization==5.0.0b2', + 'azure-mgmt-batchai==7.0.0b1', + 'azure-mgmt-batch~=17.3.0', + 'azure-mgmt-billing==6.0.0', + 'azure-mgmt-botservice~=2.0.0b3', + 'azure-mgmt-cdn==12.0.0', + 'azure-mgmt-cognitiveservices==15.0.0b4', # Supports compute operations in private preview; consult the team before removing. + 'azure-mgmt-compute~=34.1.0', + 'azure-mgmt-containerinstance==10.2.0b1', + 'azure-mgmt-containerregistry==15.1.0b2', + 'azure-mgmt-containerregistrytasks==1.0.0b1', + 'azure-mgmt-containerservice~=41.6.0', + 'azure-mgmt-cosmosdb==10.0.0', + 'azure-mgmt-datalake-store~=1.1.0b1', + 'azure-mgmt-datamigration~=10.0.0', + 'azure-mgmt-domainregistration==1.0.0b1', + 'azure-mgmt-eventgrid==10.2.0b2', + 'azure-mgmt-eventhub~=12.0.0b1', + 'azure-mgmt-extendedlocation==1.0.0b2', + 'azure-mgmt-hdinsight==9.1.0b2', + 'azure-mgmt-imagebuilder~=1.3.0', + 'azure-mgmt-iotcentral~=10.0.0b1', + 'azure-mgmt-iothub==5.0.0b1', + 'azure-mgmt-iothubprovisioningservices==1.1.0', + 'azure-mgmt-keyvault==14.0.1', + 'azure-mgmt-loganalytics==13.0.0b4', + 'azure-mgmt-managementgroups~=1.0.0', + 'azure-mgmt-maps~=2.0.0', + 'azure-mgmt-marketplaceordering==1.1.0', + 'azure-mgmt-media~=9.0', + 'azure-mgmt-monitor~=7.0.0b1', + 'azure-mgmt-msi~=7.1.0', + 'azure-mgmt-netapp~=10.1.0', + 'azure-mgmt-policyinsights==1.1.0b4', + 'azure-mgmt-postgresqlflexibleservers==3.0.0b2', + 'azure-mgmt-privatedns~=1.0.0', + 'azure-mgmt-rdbms==10.2.0b17', + 'azure-mgmt-mysqlflexibleservers==1.1.0b3', + 'azure-mgmt-recoveryservicesbackup~=9.2.0', + 'azure-mgmt-recoveryservices~=4.1.0', + 'azure-mgmt-redhatopenshift~=3.0.0', + 'azure-mgmt-redis~=14.5.0', + 'azure-mgmt-resource==24.0.0', + 'azure-mgmt-resource-deployments==1.0.0b1', + 'azure-mgmt-resource-deploymentscripts==1.0.0b1', + 'azure-mgmt-resource-deploymentstacks==1.0.0', + 'azure-mgmt-resource-templatespecs==1.0.0b1', + 'azure-mgmt-search~=9.2', + 'azure-mgmt-security==6.0.0', + 'azure-mgmt-servicebus~=10.0.0b1', + 'azure-mgmt-servicefabricmanagedclusters==2.1.0b3', + 'azure-mgmt-servicelinker==1.2.0b3', + 'azure-mgmt-servicefabric~=2.1.0', + 'azure-mgmt-signalr==2.0.0b2', + 'azure-mgmt-sqlvirtualmachine==1.0.0b5', + 'azure-mgmt-sql==4.0.0b22', + 'azure-mgmt-storage==25.0.0', + 'azure-mgmt-synapse==2.1.0b5', + 'azure-mgmt-trafficmanager~=1.0.0', + 'azure-mgmt-web==11.0.0', + 'azure-monitor-query==1.2.0', + 'azure-storage-common~=1.4', + 'azure-storage-blob==12.29.0b1', + 'azure-storage-file-datalake==12.24.0b1', + 'azure-storage-file-share==12.25.0b1', + 'azure-storage-queue==12.16.0b1', + 'azure-synapse-accesscontrol~=0.5.0', + 'azure-synapse-artifacts~=0.22.0', + 'azure-synapse-managedprivateendpoints~=0.4.0', + 'azure-synapse-spark~=0.7.0', + 'chardet~=5.2.0', + 'colorama~=0.4.4', + # On Linux, the distribution (Ubuntu, Debian, etc) and version are checked for `az feedback` + 'distro; sys_platform == "linux"', + 'fabric~=3.2.2', + 'javaproperties~=0.5.1', + 'jsondiff~=2.0.0', + 'packaging>=20.9', + 'paramiko>=2.0.8,<6.0.0', + 'pycomposefile>=0.0.34', + 'PyGithub~=1.38', + 'PyNaCl~=1.6.2', + 'scp~=0.13.2', + 'semver~=3.0', + 'six>=1.10.0', # six is still used by countless extensions + 'sshtunnel~=0.1.4', + # Even though knack already depends on tabulate, profile module directly uses it for interactive subscription + # selection + 'tabulate', + 'urllib3', + 'websocket-client~=1.8.0', + 'xmltodict~=0.12', +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false +script-files = ["az", "az.completion.sh", "az.bat", "azps.ps1"] + +[tool.setuptools.packages.find] +namespaces = false +exclude = ["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"] + +[tool.setuptools.package-data] +"azure.cli.command_modules.acr" = ["*.json"] +"azure.cli.command_modules.botservice" = ["*.json", "*.config"] +"azure.cli.command_modules.monitor.operations" = ["autoscale-parameters-template.json"] +"azure.cli.command_modules.servicefabric" = [ + "template/windows/template.json", + "template/windows/parameter.json", + "template/linux/template.json", + "template/linux/parameter.json", + "template/service/template.json", + "template/service/parameter.json", +] +"azure.cli.command_modules.appservice" = [ + "resources/WebappRuntimeStacks.json", + "resources/GenerateRandomAppNames.json", +] +"azure.cli.command_modules.rdbms" = [ + "*.json", + "randomname/adjectives.txt", + "randomname/nouns.txt", + "templates/mysql_githubaction_template.yaml", +] +"azure.cli.command_modules.mysql" = [ + "random/adjectives.txt", + "random/nouns.txt", +] +"azure.cli.command_modules.postgresql" = [ + "*.json", + "randomname/adjectives.txt", + "randomname/nouns.txt", + "templates/postgresql_githubaction_template.yaml", +] + +[tool.setuptools.cmdclass] +build_py = "azure_cli_bdist_wheel.azure_cli_build_py" \ No newline at end of file diff --git a/src/azure-cli/requirements.py3.Darwin.txt b/src/azure-cli/requirements.py3.Darwin.txt index 98d834ad7f5..6ddef11e4f7 100644 --- a/src/azure-cli/requirements.py3.Darwin.txt +++ b/src/azure-cli/requirements.py3.Darwin.txt @@ -114,7 +114,7 @@ msal-extensions==1.3.1 msal==1.36.0 msrest==0.7.1 oauthlib==3.2.2 -packaging==25.0 +packaging==26.3 paramiko==5.0.0 pbr==7.0.3 pkginfo==1.12.1.2 diff --git a/src/azure-cli/requirements.py3.Linux.txt b/src/azure-cli/requirements.py3.Linux.txt index b69fe9c2582..71f6b93542d 100644 --- a/src/azure-cli/requirements.py3.Linux.txt +++ b/src/azure-cli/requirements.py3.Linux.txt @@ -115,7 +115,7 @@ msal-extensions==1.3.1 msal==1.36.0 msrest==0.7.1 oauthlib==3.2.2 -packaging==25.0 +packaging==26.3 paramiko==5.0.0 pbr==7.0.3 pkginfo==1.12.1.2 diff --git a/src/azure-cli/requirements.py3.windows.txt b/src/azure-cli/requirements.py3.windows.txt index 8944cbb9483..4db4e49376a 100644 --- a/src/azure-cli/requirements.py3.windows.txt +++ b/src/azure-cli/requirements.py3.windows.txt @@ -114,7 +114,7 @@ msal-extensions==1.3.1 msal[broker]==1.36.0 msrest==0.7.1 oauthlib==3.2.2 -packaging==25.0 +packaging==26.3 paramiko==5.0.0 pbr==7.0.3 pkginfo==1.12.1.2 diff --git a/src/azure-cli/setup.py b/src/azure-cli/setup.py deleted file mode 100644 index 6279aa819f4..00000000000 --- a/src/azure-cli/setup.py +++ /dev/null @@ -1,223 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup, find_packages -import sys - -try: - from azure_cli_bdist_wheel import cmdclass -except ImportError: - import logging - - logging.warning("Wheel is not available, disabling bdist_wheel hook") - cmdclass = {} - -VERSION = "2.90.0" -# If we have source, validate that our version numbers match -# This should prevent uploading releases with mismatched versions. -try: - with open('azure/cli/__main__.py', 'r', encoding='utf-8') as f: - content = f.read() -except OSError: - pass -else: - import re - - m = re.search(r'__version__\s*=\s*[\'"](.+?)[\'"]', content) - if not m: - print('Could not find __version__ in azure/cli/__main__.py') - sys.exit(1) - if m.group(1) != VERSION: - print('Expected __version__ = "{}"; found "{}"'.format(VERSION, m.group(1))) - sys.exit(1) - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', -] - -DEPENDENCIES = [ - "antlr4-python3-runtime~=4.13.1", - 'azure-appconfiguration==1.10.0b1', - 'azure-ai-projects~=1.0.0', - 'azure-batch==15.0.0b1', - 'azure-cli-core=={}'.format(VERSION), - 'azure-cosmos~=3.0,>=3.0.2', - 'azure-data-tables==12.4.0', - 'azure-datalake-store~=1.0.1', - 'azure-keyvault-administration==4.8.0b2', - 'azure-keyvault-certificates==4.7.0', - 'azure-keyvault-keys==4.12.0b2', - 'azure-keyvault-secrets==4.7.0', - 'azure-keyvault-securitydomain==1.0.0b1', - 'azure-mgmt-advisor==9.0.0', - 'azure-mgmt-apimanagement==4.0.0', - 'azure-mgmt-appconfiguration==6.0.0b2', - 'azure-mgmt-appcontainers==2.0.0', - 'azure-mgmt-applicationinsights~=1.0.0', - 'azure-mgmt-authorization==5.0.0b2', - 'azure-mgmt-batchai==7.0.0b1', - 'azure-mgmt-batch~=17.3.0', - 'azure-mgmt-billing==6.0.0', - 'azure-mgmt-botservice~=2.0.0b3', - 'azure-mgmt-cdn==12.0.0', - 'azure-mgmt-cognitiveservices==15.0.0b4', # 15.0.0b4 version supports compute operations which is in PrPr, pls don't remove it without consulting our team - 'azure-mgmt-compute~=34.1.0', - 'azure-mgmt-containerinstance==10.2.0b1', - 'azure-mgmt-containerregistry==15.1.0b2', - 'azure-mgmt-containerregistrytasks==1.0.0b1', - 'azure-mgmt-containerservice~=41.6.0', - 'azure-mgmt-cosmosdb==10.0.0', - 'azure-mgmt-datalake-store~=1.1.0b1', - 'azure-mgmt-datamigration~=10.0.0', - 'azure-mgmt-domainregistration==1.0.0b1', - 'azure-mgmt-eventgrid==10.2.0b2', - 'azure-mgmt-eventhub~=12.0.0b1', - 'azure-mgmt-extendedlocation==1.0.0b2', - 'azure-mgmt-hdinsight==9.1.0b2', - 'azure-mgmt-imagebuilder~=1.3.0', - 'azure-mgmt-iotcentral~=10.0.0b1', - 'azure-mgmt-iothub==5.0.0b1', - 'azure-mgmt-iothubprovisioningservices==1.1.0', - 'azure-mgmt-keyvault==14.0.1', - 'azure-mgmt-loganalytics==13.0.0b4', - 'azure-mgmt-managementgroups~=1.0.0', - 'azure-mgmt-maps~=2.0.0', - 'azure-mgmt-marketplaceordering==1.1.0', - 'azure-mgmt-media~=9.0', - 'azure-mgmt-monitor~=7.0.0b1', - 'azure-mgmt-msi~=7.1.0', - 'azure-mgmt-netapp~=10.1.0', - 'azure-mgmt-policyinsights==1.1.0b4', - 'azure-mgmt-postgresqlflexibleservers==3.0.0b2', - 'azure-mgmt-privatedns~=1.0.0', - 'azure-mgmt-rdbms==10.2.0b17', - 'azure-mgmt-mysqlflexibleservers==1.1.0b3', - 'azure-mgmt-recoveryservicesbackup~=9.2.0', - 'azure-mgmt-recoveryservices~=4.1.0', - 'azure-mgmt-redhatopenshift~=3.0.0', - 'azure-mgmt-redis~=14.5.0', - 'azure-mgmt-resource==24.0.0', - 'azure-mgmt-resource-deployments==1.0.0b1', - 'azure-mgmt-resource-deploymentscripts==1.0.0b1', - 'azure-mgmt-resource-deploymentstacks==1.0.0', - 'azure-mgmt-resource-templatespecs==1.0.0b1', - 'azure-mgmt-search~=9.2', - 'azure-mgmt-security==6.0.0', - 'azure-mgmt-servicebus~=10.0.0b1', - 'azure-mgmt-servicefabricmanagedclusters==2.1.0b3', - 'azure-mgmt-servicelinker==1.2.0b3', - 'azure-mgmt-servicefabric~=2.1.0', - 'azure-mgmt-signalr==2.0.0b2', - 'azure-mgmt-sqlvirtualmachine==1.0.0b5', - 'azure-mgmt-sql==4.0.0b22', - 'azure-mgmt-storage==25.0.0', - 'azure-mgmt-synapse==2.1.0b5', - 'azure-mgmt-trafficmanager~=1.0.0', - 'azure-mgmt-web==11.0.0', - 'azure-monitor-query==1.2.0', - 'azure-storage-common~=1.4', - 'azure-storage-blob==12.29.0b1', - 'azure-storage-file-datalake==12.24.0b1', - 'azure-storage-file-share==12.25.0b1', - 'azure-storage-queue==12.16.0b1', - 'azure-synapse-accesscontrol~=0.5.0', - 'azure-synapse-artifacts~=0.22.0', - 'azure-synapse-managedprivateendpoints~=0.4.0', - 'azure-synapse-spark~=0.7.0', - 'chardet~=5.2.0', - 'colorama~=0.4.4', - # On Linux, the distribution (Ubuntu, Debian, etc) and version are checked for `az feedback` - 'distro; sys_platform == "linux"', - 'fabric~=3.2.2', - 'javaproperties~=0.5.1', - 'jsondiff~=2.0.0', - 'packaging>=20.9', - 'paramiko>=2.0.8,<6.0.0', - 'pycomposefile>=0.0.34', - 'PyGithub~=1.38', - 'PyNaCl~=1.6.2', - 'scp~=0.13.2', - 'semver~=3.0', - 'six>=1.10.0', # six is still used by countless extensions - 'sshtunnel~=0.1.4', - # Even though knack already depends on tabulate, profile module directly uses it for interactive subscription - # selection - 'tabulate', - 'urllib3', - 'websocket-client~=1.8.0', - 'xmltodict~=0.12' -] - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() - -setup( - name='azure-cli', - version=VERSION, - description='Microsoft Azure Command-Line Tools', - long_description=README, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - scripts=[ - 'az', - 'az.completion.sh', - 'az.bat', - 'azps.ps1' - ], - packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"]), - install_requires=DEPENDENCIES, - python_requires='>=3.10.0', - package_data={ - 'azure.cli.command_modules.acr': ['*.json'], - 'azure.cli.command_modules.botservice': ['*.json', '*.config'], - 'azure.cli.command_modules.monitor.operations': ['autoscale-parameters-template.json'], - 'azure.cli.command_modules.servicefabric': [ - 'template/windows/template.json', - 'template/windows/parameter.json', - 'template/linux/template.json', - 'template/linux/parameter.json', - 'template/service/template.json', - 'template/service/parameter.json' - ], - 'azure.cli.command_modules.appservice': [ - 'resources/WebappRuntimeStacks.json', - 'resources/GenerateRandomAppNames.json' - ], - 'azure.cli.command_modules.rdbms': [ - '*.json', - 'randomname/adjectives.txt', - 'randomname/nouns.txt', - 'templates/mysql_githubaction_template.yaml' - ], - 'azure.cli.command_modules.mysql': [ - 'random/adjectives.txt', - 'random/nouns.txt' - ], - 'azure.cli.command_modules.postgresql': [ - '*.json', - 'randomname/adjectives.txt', - 'randomname/nouns.txt', - 'templates/postgresql_githubaction_template.yaml' - ] - }, - cmdclass=cmdclass -) diff --git a/tools/automation/tests/test_packaging_migration.py b/tools/automation/tests/test_packaging_migration.py new file mode 100644 index 00000000000..f7dc9cd1079 --- /dev/null +++ b/tools/automation/tests/test_packaging_migration.py @@ -0,0 +1,321 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Offline packaging regression tests; subprocesses and PyPI access are mocked.""" + +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from automation.utilities import packaging, path as package_paths +from automation.tests import verify_package_versions as versions +from automation.tests import verify_readme_history as readme +from automation.verify import default_modules + + +RUNTIME_PATHS = { + 'azure-cli': 'azure/cli/__main__.py', + 'azure-cli-core': 'azure/cli/core/__init__.py', +} + + +class PackageFixture(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='cli packaging tests ') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + def write(self, relative_path, text): + target = self.root / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding='utf-8') + return target + + def package(self, name, version='1.2.3', parent='src'): + folder = Path(parent) / name + self.write(folder / 'pyproject.toml', + '[project]\nname = "{}"\nversion = "{}"\n'.format(name, version)) + if name in RUNTIME_PATHS: + self.write(folder / RUNTIME_PATHS[name], + 'raise AssertionError("Runtime must not be imported")\n' + '__version__ = "{}"\n'.format(version)) + return self.root / folder + + +class PackageVersionTests(PackageFixture): + def test_static_metadata_for_all_four_distributions_without_execution(self): + with mock.patch.object(packaging.subprocess, 'check_output') as execute: + for name in ('azure-cli', 'azure-cli-core', 'azure-cli-telemetry', 'azure-cli-testsdk'): + with self.subTest(name=name): + folder = self.package(name) + self.write(folder / 'setup.py', 'raise AssertionError("Do not execute")\n') + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + execute.assert_not_called() + + def test_metadata_read_uses_toml_not_text_matching(self): + folder = self.package('azure-cli-telemetry') + self.write(folder / 'pyproject.toml', + '# version = "wrong"\n[tool.example]\nversion = "also-wrong"\n' + "[project]\nname = 'azure-cli-telemetry'\nversion = '1.2.3' # comment\n") + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + + def test_tomli_fallback_when_tomllib_is_unavailable(self): + spec = importlib.util.spec_from_file_location('packaging_fallback', packaging.__file__) + module = importlib.util.module_from_spec(spec) + # Exercise the import fallback without requiring both parsers installed. + with mock.patch.dict(sys.modules, {'tomllib': None, 'tomli': packaging.tomllib}): + spec.loader.exec_module(module) + self.assertEqual(module.get_package_version(self.package('azure-cli-testsdk')), '1.2.3') + + def test_invalid_modern_metadata_never_falls_back_to_setup(self): + folder = self.package('azure-cli-testsdk') + self.write(folder / 'setup.py', '# legacy metadata must not hide invalid TOML\n') + cases = ( + ('[project\n', 'Invalid TOML'), + ('[project]\nname = "azure-cli-testsdk"\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\nversion = 123\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\nversion = ""\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\ndynamic = ["version"]\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\nversion = "1.2.3"\n' + 'dynamic = ["version"]\n', 'static'), + ('[project]\nversion = "1.2.3"\n', 'name'), + ) + with mock.patch.object(packaging.subprocess, 'check_output') as execute: + for content, message in cases: + with self.subTest(content=content): + self.write(folder / 'pyproject.toml', content) + with self.assertRaisesRegex(ValueError, message): + packaging.get_package_version(folder) + execute.assert_not_called() + + def test_runtime_mismatch_reports_both_versions_and_paths(self): + for name, runtime_file in RUNTIME_PATHS.items(): + with self.subTest(name=name): + folder = self.package(name) + self.write(folder / runtime_file, '__version__ = "9.8.7"\n') + with self.assertRaises(ValueError) as caught: + packaging.get_package_version(folder) + for detail in (name, '1.2.3', '9.8.7', str(folder / 'pyproject.toml'), + str(folder / runtime_file)): + self.assertIn(detail, str(caught.exception)) + + def test_runtime_must_be_one_top_level_literal(self): + folder = self.package('azure-cli-core') + for content in ( + '# __version__ = "1.2.3"\n', + '__version__ = get_version()\n', + '__version__ = 123\n', + '__version__: str\n', + '__version__ = "1.2.3"\n__version__ = "1.2.3"\n', + 'def nested():\n __version__ = "1.2.3"\n'): + with self.subTest(content=content): + self.write(folder / RUNTIME_PATHS['azure-cli-core'], content) + with self.assertRaisesRegex(ValueError, 'top-level string literal __version__'): + packaging.get_package_version(folder) + + def test_annotated_runtime_literal_is_supported(self): + folder = self.package('azure-cli-core') + self.write(folder / RUNTIME_PATHS['azure-cli-core'], '__version__: str = "1.2.3"\n') + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + + def test_missing_or_invalid_runtime_source_is_not_silently_skipped(self): + folder = self.package('azure-cli') + runtime_file = folder / RUNTIME_PATHS['azure-cli'] + runtime_file.unlink() + with self.assertRaisesRegex(ValueError, 'Unable to read runtime __version__'): + packaging.get_package_version(folder) + self.write(runtime_file, '__version__ = (\n') + with self.assertRaisesRegex(ValueError, 'Unable to read runtime __version__'): + packaging.get_package_version(folder) + + def test_legacy_fallback_uses_current_interpreter_and_package_directory(self): + folder = self.root / 'legacy package' + self.write(folder / 'setup.py', '# historical setup\n') + with mock.patch.object(packaging.subprocess, 'check_output', return_value='1.2.3\n') as execute: + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + execute.assert_called_once_with( + [sys.executable, 'setup.py', '--version'], cwd=str(folder), universal_newlines=True) + + def test_missing_package_has_useful_error(self): + with self.assertRaisesRegex(FileNotFoundError, 'No pyproject.toml or setup.py'): + packaging.get_package_version(self.root) + + def test_historical_base_repo_comparison_retains_legacy_fallback(self): + current = self.root / 'current' + base = self.root / 'historical' + relative = Path('src') / 'azure-cli' + self.write(base / relative / 'setup.py', '# historical setup\n') + with mock.patch.object(versions, 'get_repo_root', return_value=str(current)), \ + mock.patch.object(packaging.subprocess, 'check_output', return_value='1.2.3\n') as execute: + self.assertTrue(versions.version_in_base_repo( + str(base), str(current / relative), 'azure-cli', '1.2.3')) + execute.assert_called_once_with( + [sys.executable, 'setup.py', '--version'], cwd=str(base / relative), universal_newlines=True) + + def test_version_validator_checks_runtime_before_contacting_pypi(self): + folder = self.package('azure-cli') + self.write(folder / RUNTIME_PATHS['azure-cli'], '__version__ = "9.8.7"\n') + with mock.patch.object(versions, 'is_available_on_pypi') as pypi: + with self.assertRaisesRegex(ValueError, 'Version mismatch'): + versions.check_package_version('azure-cli', str(folder)) + pypi.assert_not_called() + + +class PackageDiscoveryTests(PackageFixture): + def test_core_discovery_includes_four_pyprojects_legacy_and_deduplicates(self): + names = ('azure-cli', 'azure-cli-core', 'azure-cli-telemetry', 'azure-cli-testsdk') + expected = [(name, str(self.package(name))) for name in names] + self.write('src/azure-cli/setup.py', '# transitional package\n') + self.write('src/legacy/setup.py', '# legacy\n') + expected.append(('legacy', str(self.root / 'src' / 'legacy'))) + self.write('src/not-a-package/README.rst', 'Not a package\n') + (self.root / 'src' / 'not-a-package' / 'pyproject.toml').mkdir() + with mock.patch.object(package_paths, 'get_repo_root', return_value=str(self.root)): + self.assertEqual(sorted(package_paths.get_core_modules_paths()), sorted(expected)) + + def test_command_discovery_preserves_prefix_option_and_combined_paths(self): + core = self.package('azure-cli-core') + command = self.package('azure-cli-modern', parent='src/command_modules') + self.write(command / 'setup.py', '# transitional package\n') + legacy = self.root / 'src' / 'command_modules' / 'azure-cli-legacy' + self.write(legacy / 'setup.py', '# legacy\n') + self.package('not-a-command', parent='src/command_modules') + with mock.patch.object(package_paths, 'get_repo_root', return_value=str(self.root)): + self.assertEqual(list(package_paths.get_command_modules_paths()), + [('legacy', str(legacy)), ('modern', str(command))]) + prefixed = [('azure-cli-legacy', str(legacy)), ('azure-cli-modern', str(command))] + self.assertEqual(list(package_paths.get_command_modules_paths(include_prefix=True)), prefixed) + self.assertEqual(package_paths.get_all_module_paths(), [('azure-cli-core', str(core))] + prefixed) + + def test_default_modules_points_to_main_pyproject(self): + self.assertEqual(Path(default_modules.AZURE_CLI_PYPROJECT), + Path(default_modules.AZURE_CLI_PATH) / 'pyproject.toml') + + +class ReleaseVersionTests(PackageFixture): + def setUp(self): + super().setUp() + script = Path(__file__).resolve().parents[3] / 'scripts/ci/check_package_versions.py' + spec = importlib.util.spec_from_file_location('check_package_versions', script) + self.checker = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.checker) + self.checker.ROOT = self.root + for name in self.checker.PACKAGES: + self.package(name) + with (self.root / 'src/azure-cli/pyproject.toml').open('a', encoding='utf-8') as stream: + stream.write('dependencies = ["azure-cli-core==1.2.3"]\n') + + def test_matching_versions_pass(self): + self.checker.main() + + def test_missing_distribution_fails(self): + (self.root / 'src/azure-cli-testsdk/pyproject.toml').unlink() + with self.assertRaises(FileNotFoundError): + self.checker.main() + + def test_mismatched_core_dependency_fails(self): + self.package('azure-cli-core', version='1.2.4') + with self.assertRaisesRegex(ValueError, 'including the dependency'): + self.checker.main() + + def test_nightly_versions_and_dependency_are_synchronized(self): + for name in self.checker.PACKAGES: + self.package(name, version='1.2.3.dev20260918000000') + with (self.root / 'src/azure-cli/pyproject.toml').open('a', encoding='utf-8') as stream: + stream.write('dependencies = ["azure-cli-core==1.2.3.dev20260918000000"]\n') + self.checker.main() + + +class ReadmeValidationTests(PackageFixture): + def history(self, folder, version='1.2.3'): + self.write(folder / 'HISTORY.rst', + 'Release History\n===============\n\n{}\n{}\n\n* Changes.\n\n' + '0.0.1\n-----\n\n* Initial release.\n'.format( + version, '-' * len(version))) + + def test_history_uses_static_source_version(self): + folder = self.package('azure-cli-telemetry') + self.history(folder) + with mock.patch.object(packaging.subprocess, 'check_output') as execute: + self.assertTrue(readme.check_history_headings(str(folder))) + self.history(folder, '9.8.7') + self.assertFalse(readme.check_history_headings(str(folder))) + execute.assert_not_called() + + def test_history_retains_runtime_mismatch_validation(self): + folder = self.package('azure-cli-core') + self.history(folder) + self.write(folder / RUNTIME_PATHS['azure-cli-core'], '__version__ = "9.8.7"\n') + with self.assertRaisesRegex(ValueError, 'Version mismatch'): + readme.check_history_headings(str(folder)) + + def test_build_and_twine_use_clean_temporary_copy_and_leave_source_untouched(self): + folder = self.package('azure-cli-telemetry') + self.history(folder) + self.write(folder / 'README.rst', 'Original readme\n') + self.write(folder / 'azure/cli/telemetry/__init__.py', '# package\n') + ignored = ('tests/recordings/large.yaml', 'build/stale.txt', 'dist/stale.tar.gz', + '__pycache__/stale.pyc', 'old.egg-info/PKG-INFO', '.venv/large.txt') + for relative in ignored: + self.write(folder / relative, 'stale\n') + before = {p.relative_to(folder): p.read_bytes() for p in folder.rglob('*') if p.is_file()} + commands = [] + working_dirs = [] + + def execute(command, **kwargs): + commands.append(command) + source = Path(kwargs['cwd']) + working_dirs.append(source) + self.assertNotEqual(source, folder) + self.assertTrue((source / 'pyproject.toml').is_file()) + if command[2] == 'build': + self.assertEqual(command[:5], [sys.executable, '-m', 'build', '--sdist', '--outdir']) + self.assertTrue((source / 'azure/cli/telemetry/__init__.py').is_file()) + for relative in ignored: + self.assertFalse((source / relative).exists()) + (source / 'README.rst').write_text('Backend changed the copy\n', encoding='utf-8') + output = Path(command[5]) + output.mkdir() + (output / 'azure_cli_telemetry-1.2.3.tar.gz').write_bytes(b'mocked sdist') + else: + self.assertEqual(command[:5], [sys.executable, '-m', 'twine', 'check', '--strict']) + self.assertEqual(len(command), 6) + self.assertTrue(Path(command[5]).is_file()) + + with mock.patch.object(readme.subprocess, 'check_call', side_effect=execute): + self.assertTrue(readme.check_readme_render(str(folder))) + self.assertEqual(len(commands), 2) + self.assertEqual(working_dirs[0], working_dirs[1]) + self.assertFalse(working_dirs[0].parent.exists()) + after = {p.relative_to(folder): p.read_bytes() for p in folder.rglob('*') if p.is_file()} + self.assertEqual(before, after) + + def test_failed_build_missing_sdist_and_twine_failure_are_reported_and_cleaned(self): + folder = self.package('azure-cli-testsdk') + for failure in ('build', 'missing-sdist', 'twine'): + with self.subTest(failure=failure): + working_dirs = [] + + def execute(command, **kwargs): + working_dirs.append(Path(kwargs['cwd'])) + if command[2] == failure: + raise subprocess.CalledProcessError(1, command) + if command[2] == 'build' and failure != 'missing-sdist': + output = Path(command[5]) + output.mkdir() + (output / 'package-1.2.3.tar.gz').write_bytes(b'mocked sdist') + + with mock.patch.object(readme.subprocess, 'check_call', side_effect=execute) as call: + self.assertFalse(readme._check_sdist_render(str(folder))) + self.assertEqual(call.call_count, 2 if failure == 'twine' else 1) + self.assertFalse(working_dirs[0].parent.exists()) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tools/automation/tests/verify_package_versions.py b/tools/automation/tests/verify_package_versions.py index caf9cdd066a..d63bd720625 100644 --- a/tools/automation/tests/verify_package_versions.py +++ b/tools/automation/tests/verify_package_versions.py @@ -9,11 +9,10 @@ import subprocess from ..utilities.path import get_all_module_paths, get_repo_root +from ..utilities.packaging import get_package_version from ..utilities.display import print_heading from ..utilities.pypi import is_available_on_pypi -SETUP_PY_NAME = 'setup.py' - def is_unreleased_version(mod_name, mod_version): if is_available_on_pypi(mod_name, mod_version): @@ -67,8 +66,7 @@ def version_in_base_repo(base_repo, mod_path, mod_name, mod_version): def _get_mod_version(mod_path): - return subprocess.check_output(['python', 'setup.py', '--version'], cwd=mod_path, - universal_newlines=True).strip() + return get_package_version(mod_path) def check_package_version(mod_name, mod_path, base_repo=None, base_tag=None): diff --git a/tools/automation/tests/verify_readme_history.py b/tools/automation/tests/verify_readme_history.py index c18ce39d88a..ed30fe71082 100644 --- a/tools/automation/tests/verify_readme_history.py +++ b/tools/automation/tests/verify_readme_history.py @@ -9,9 +9,13 @@ import sys import argparse import subprocess +from pathlib import Path +import shutil +import tempfile from docutils import core, io from ..utilities.path import get_all_module_paths +from ..utilities.packaging import get_package_version from ..utilities.display import print_heading HISTORY_NAME = 'HISTORY.rst' @@ -63,19 +67,39 @@ def check_history_headings(mod_path): print("Unable to get versions from {}. Check formatting. e.g. there should be a new line after the 'Release History' heading.".format(history_path)) return False first_version_history = all_versions[0] - actual_version = subprocess.check_output('python setup.py --version'.split(), cwd=mod_path, universal_newlines=True) - actual_version = actual_version.strip() + actual_version = get_package_version(mod_path) if first_version_history != actual_version: - print("The topmost version in {} does not match version {} defined in setup.py.".format(history_path, actual_version)) + print("The topmost version in {} does not match source package version {}.".format( + history_path, actual_version)) return False return True +def _check_sdist_render(mod_path): + # Build only this package, not the repository. Omit tests/recordings and stale + # build metadata: they are unnecessary for README validation and can be huge. + # A real copy (not hard links) keeps backend writes away from the source tree. + with tempfile.TemporaryDirectory(prefix='cli-readme-') as temp_dir: + source_dir = Path(temp_dir) / 'source' + dist_dir = Path(temp_dir) / 'dist' + shutil.copytree(mod_path, source_dir, ignore=shutil.ignore_patterns( + '.git', '.venv', 'venv', 'env', '__pycache__', '*.pyc', '*.pyo', + '*.egg-info', '*.dist-info', '.pytest_cache', 'build', 'dist', 'tests', 'recordings')) + if not exec_command([sys.executable, '-m', 'build', '--sdist', '--outdir', str(dist_dir)], + cwd=str(source_dir)): + return False + artifacts = sorted(str(path) for path in dist_dir.glob('*.tar.gz')) + if not artifacts: + print('No sdist produced for {}'.format(mod_path), file=sys.stderr) + return False + return exec_command([sys.executable, '-m', 'twine', 'check', '--strict'] + artifacts, + cwd=str(source_dir)) + + def check_readme_render(mod_path): - checks = [] - checks.append(exec_command('python setup.py check -r -s', cwd=mod_path)) - checks.append(check_history_headings(mod_path)) - return all(checks) + history_ok = check_history_headings(mod_path) + render_ok = _check_sdist_render(mod_path) + return history_ok and render_ok def verify_all(): @@ -93,7 +117,7 @@ def verify_all(): print('The following modules have invalid README/HISTORYs:') print('\n'.join(failed_mods)) print('See above for the full warning/errors') - print('note: Line numbers in the errors map to the long_description of your setup.py.') + print('note: Rendering error line numbers refer to the built package description.') sys.exit(1) else: print('Verified READMEs of all modules successfully.', file=sys.stderr) @@ -106,7 +130,7 @@ def verify_one(mod_name): if not res: print_heading('Error whilst verifying README/HISTORY of {}!'.format(mod_name)) print('See above for the full warning/errors.') - print('note: Line numbers in the errors map to the long_description of your setup.py.') + print('note: Rendering error line numbers refer to the built package description.') sys.exit(1) diff --git a/tools/automation/utilities/packaging.py b/tools/automation/utilities/packaging.py new file mode 100644 index 00000000000..abbf3f05b7e --- /dev/null +++ b/tools/automation/utilities/packaging.py @@ -0,0 +1,92 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Read source package versions without importing the CLI or invoking a build.""" + +import ast +from pathlib import Path +import subprocess +import sys +import tokenize + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib + + +_RUNTIME_VERSION_FILES = { + 'azure-cli': 'azure/cli/__main__.py', + 'azure-cli-core': 'azure/cli/core/__init__.py', +} + + +def _check_runtime_version(package_path, name, version): + relative_path = _RUNTIME_VERSION_FILES.get(name) + if relative_path is None: + return + + runtime_path = package_path / relative_path + try: + with tokenize.open(runtime_path) as source: + tree = ast.parse(source.read(), filename=str(runtime_path)) + except (OSError, SyntaxError) as error: + raise ValueError('Unable to read runtime __version__ for {} from {}: {}'.format( + name, runtime_path, error)) from error + + values = [] + for statement in tree.body: + if isinstance(statement, ast.Assign): + targets = statement.targets + elif isinstance(statement, ast.AnnAssign): + targets = [statement.target] + else: + continue + if any(isinstance(target, ast.Name) and target.id == '__version__' for target in targets): + values.append(statement.value) + + if (len(values) != 1 or not isinstance(values[0], ast.Constant) + or not isinstance(values[0].value, str)): + raise ValueError('Expected one top-level string literal __version__ in {}'.format(runtime_path)) + runtime_version = values[0].value + if runtime_version != version: + raise ValueError( + 'Version mismatch for {}: {} [project].version is {!r}, but {} __version__ is {!r}. ' + 'Update both source versions together.'.format( + name, package_path / 'pyproject.toml', version, runtime_path, runtime_version)) + + +def get_package_version(package_path): + """Prefer static PEP 621 metadata; retain setup.py for historical checkouts. + + Invalid modern metadata must fail rather than silently falling back to setup.py. + The main/core runtime literals are checked here so every source-version consumer + retains the validation previously performed by their setup scripts. + """ + package_path = Path(package_path) + pyproject_path = package_path / 'pyproject.toml' + if pyproject_path.is_file(): + try: + with pyproject_path.open('rb') as source: + metadata = tomllib.load(source) + except ValueError as error: + raise ValueError('Invalid TOML in {}: {}'.format(pyproject_path, error)) from error + project = metadata.get('project', {}) + name = project.get('name') + version = project.get('version') + if not isinstance(name, str) or not name.strip(): + raise ValueError('Expected a non-empty [project].name in {}'.format(pyproject_path)) + if (not isinstance(version, str) or not version.strip() + or 'version' in project.get('dynamic', [])): + raise ValueError('Expected a static, non-empty [project].version in {}'.format(pyproject_path)) + _check_runtime_version(package_path, name, version) + return version + + setup_path = package_path / 'setup.py' + if not setup_path.is_file(): + raise FileNotFoundError('No pyproject.toml or setup.py found in {}'.format(package_path)) + return subprocess.check_output( + [sys.executable, 'setup.py', '--version'], cwd=str(package_path), + universal_newlines=True).strip() \ No newline at end of file diff --git a/tools/automation/utilities/path.py b/tools/automation/utilities/path.py index a951b401c45..f7790688746 100644 --- a/tools/automation/utilities/path.py +++ b/tools/automation/utilities/path.py @@ -44,9 +44,8 @@ def get_extensions_paths(include_prefix=False): def get_command_modules_paths(include_prefix=False): - glob_pattern = os.path.normcase('/src/command_modules/{}*/setup.py'.format(COMMAND_MODULE_PREFIX)) - for path in glob.glob(get_repo_root() + glob_pattern): - folder = os.path.dirname(path) + folder_pattern = os.path.join(get_repo_root(), 'src', 'command_modules', COMMAND_MODULE_PREFIX + '*') + for folder in _get_package_paths(folder_pattern): name = os.path.basename(folder) if not include_prefix: name = name[len(COMMAND_MODULE_PREFIX):] @@ -68,8 +67,17 @@ def get_core_modules_paths_with_tests(profile): def get_core_modules_paths(): - for path in glob.glob(get_repo_root() + os.path.normcase('/src/*/setup.py')): - yield os.path.basename(os.path.dirname(path)), os.path.dirname(path) + for folder in _get_package_paths(os.path.join(get_repo_root(), 'src', '*')): + yield os.path.basename(folder), folder + + +def _get_package_paths(folder_pattern): + """Discover modern and legacy packages once, including transitional trees.""" + folders = set() + for filename in ('pyproject.toml', 'setup.py'): + folders.update(os.path.dirname(path) for path in glob.glob(os.path.join(folder_pattern, filename)) + if os.path.isfile(path)) + yield from sorted(folders) def get_module_paths_with_tests(modules, profile): diff --git a/tools/automation/verify/default_modules.py b/tools/automation/verify/default_modules.py index 56d8dd56d32..0473d138778 100644 --- a/tools/automation/verify/default_modules.py +++ b/tools/automation/verify/default_modules.py @@ -13,7 +13,7 @@ from automation.utilities.display import print_heading AZURE_CLI_PATH = os.path.join(get_repo_root(), 'src', 'azure-cli') -AZURE_CLI_SETUP_PY = os.path.join(AZURE_CLI_PATH, 'setup.py') +AZURE_CLI_PYPROJECT = os.path.join(AZURE_CLI_PATH, 'pyproject.toml') def get_cli_dependencies(build_folder): @@ -39,13 +39,13 @@ def verify_default_modules(args): cli_deps = get_cli_dependencies(args.build_folder) all_command_modules = get_command_modules_paths(include_prefix=True) if not cli_deps: - print('Unable to get the CLI dependencies for {}'.format(AZURE_CLI_SETUP_PY), file=sys.stderr) + print('Unable to get the CLI dependencies for {}'.format(AZURE_CLI_PYPROJECT), file=sys.stderr) sys.exit(1) for modname, _ in all_command_modules: if modname not in cli_deps: - errors_list.append("{} is not included to be installed by default! Modify {}.".format(modname, AZURE_CLI_SETUP_PY)) + errors_list.append("{} is not included to be installed by default! Modify {}.".format(modname, AZURE_CLI_PYPROJECT)) if errors_list: - print_heading('Errors whilst verifying default modules list in {}!'.format(AZURE_CLI_SETUP_PY)) + print_heading('Errors whilst verifying default modules list in {}!'.format(AZURE_CLI_PYPROJECT)) print('\n'.join(errors_list), file=sys.stderr) sys.exit(1) else: diff --git a/tools/setup.py b/tools/setup.py index 356f5b8c94b..29273d9915d 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -21,6 +21,9 @@ # Until https://gitlab.com/pycqa/flake8/issues/415 is resolved, pin version of pycodestyle DEPENDENCIES = [ + 'build>=1.2.2', + 'twine>=6.1.0', + 'tomli>=2.0.1; python_version < "3.11"', 'coverage>=4.2', 'flake8==3.5.0', 'pycodestyle==2.3.1', From 00ab64cf1b2f862fdc0e0b20fafee2e1795d5ba3 Mon Sep 17 00:00:00 2001 From: aryasadeghi1 Date: Tue, 29 Sep 2026 14:25:11 +1000 Subject: [PATCH 2/3] ci: pin azdev validation candidates to GitHub commit Install azdev and azure-cli-diff-tool together from the same immutable candidate revision until the packages are released. Apply the pins to shared setup and current live-test jobs. --- .azure-pipelines/templates/automation_test.yml | 4 ++-- .azure-pipelines/templates/azdev_setup.yml | 12 +++++++----- .github/actions/env-setup/action.yml | 6 +++++- scripts/live_test/CLITest.yml | 11 ++++++++--- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines/templates/automation_test.yml b/.azure-pipelines/templates/automation_test.yml index 06782c2f330..742745a5ca6 100644 --- a/.azure-pipelines/templates/automation_test.yml +++ b/.azure-pipelines/templates/automation_test.yml @@ -29,10 +29,10 @@ parameters: default: '' - name: AzdevPackage type: string - default: 'azdev==0.2.14' + default: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' - name: DiffToolPackage type: string - default: '' + default: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' steps: - task: UsePythonVersion@0 diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index e07b3bfcd73..454998cc268 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -8,13 +8,14 @@ parameters: - name: EnableCompactAAZ type: boolean default: false - # Override with a candidate wheel path when validating before the azdev release. + # Temporary immutable candidates until the validated packages are released. + # Keep both requirements on the same commit; overrides may also be wheel paths. - name: AzdevPackage type: string - default: 'azdev==0.2.14' + default: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' - name: DiffToolPackage type: string - default: '' + default: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' steps: - bash: | set -ev @@ -24,10 +25,11 @@ steps: . env/bin/activate python -m pip install -U pip "setuptools>=78.1.1" + packages=("$AZDEV_PACKAGE") if [ -n "$DIFF_TOOL_PACKAGE" ]; then - pip install --upgrade "$DIFF_TOOL_PACKAGE" + packages+=("$DIFF_TOOL_PACKAGE") fi - pip install --upgrade "$AZDEV_PACKAGE" + python -m pip install --upgrade "${packages[@]}" azdev --version if [ -z "$CLI_EXT_REPO_PATH" ]; then diff --git a/.github/actions/env-setup/action.yml b/.github/actions/env-setup/action.yml index 5e2a9226004..029534eb5f8 100644 --- a/.github/actions/env-setup/action.yml +++ b/.github/actions/env-setup/action.yml @@ -23,13 +23,17 @@ runs: python-version: "3.12" - name: Install azdev shell: bash + # Temporary immutable candidates; neither package needs a PyPI release. + env: + AZDEV_PACKAGE: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' + DIFF_TOOL_PACKAGE: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' run: | python -m pip install --upgrade pip set -ev python -m venv env chmod +x env/bin/activate source ./env/bin/activate - pip install "azdev==0.2.14" + python -m pip install "$AZDEV_PACKAGE" "$DIFF_TOOL_PACKAGE" azdev --version cd ../ azdev setup -c azure-cli --debug diff --git a/scripts/live_test/CLITest.yml b/scripts/live_test/CLITest.yml index 69ff49977ab..75e8e1fd513 100644 --- a/scripts/live_test/CLITest.yml +++ b/scripts/live_test/CLITest.yml @@ -13,6 +13,11 @@ trigger: variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml +# Temporary immutable candidates, matching the shared CI setup. +- name: AzdevPackage + value: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' +- name: DiffToolPackage + value: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' parameters: - name: maxParallel @@ -558,7 +563,7 @@ stages: source env/bin/activate # TODO: support pytest-html >= 4.0.0 - pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures + python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures azdev setup -c azure-cli -r azure-cli-extensions @@ -1136,7 +1141,7 @@ stages: export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES # TODO: support pytest-html >= 4.0.0 - pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures + python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures azdev setup -c azure-cli -r azure-cli-extensions @@ -1712,7 +1717,7 @@ stages: source env/Scripts/activate # TODO: support pytest-html >= 4.0.0 - pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures + python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures azdev setup -c azure-cli -r azure-cli-extensions From dc250eb2f4f5a19cb8ff6c0bf888bd2014d31a6f Mon Sep 17 00:00:00 2001 From: aryasadeghi1 Date: Tue, 29 Sep 2026 16:01:58 +1000 Subject: [PATCH 3/3] ci: handle shortened JUnit classnames after packaging migration --- azure-pipelines.yml | 2 +- scripts/ci/automation_full_test.py | 120 ++++++------ .../automation/tests/test_pipeline_results.py | 178 ++++++++++++++++++ 3 files changed, 246 insertions(+), 54 deletions(-) create mode 100644 tools/automation/tests/test_pipeline_results.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d3cc541f385..29c279685f2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -559,7 +559,7 @@ jobs: - bash: | set -e python -m pip install docutils requests packaging tomli - python -B -m unittest discover -s tools/automation/tests -t tools -p test_packaging_migration.py -v + python -B -m unittest discover -s tools/automation/tests -t tools -p 'test_*.py' -v python -B scripts/ci/check_package_versions.py displayName: Validate source metadata and packaging automation diff --git a/scripts/ci/automation_full_test.py b/scripts/ci/automation_full_test.py index e4dc30cae80..1ae10f55216 100644 --- a/scripts/ci/automation_full_test.py +++ b/scripts/ci/automation_full_test.py @@ -9,6 +9,7 @@ import json import logging import os +from pathlib import Path import subprocess import sys import time @@ -423,60 +424,73 @@ def build_pipeline_result(): return pipeline_result -def get_pipeline_result(test_result_fp, pipeline_result): - tree = ET.parse(test_result_fp) - root = tree.getroot() - for testsuite in root: - for testcase in testsuite: - # ['azure', 'cli', 'command_modules', 'network', 'tests', 'latest', 'test_network_commands', 'NetworkNicScenarioTest'] - # ['src', 'azure-cli', 'azure', 'cli', 'command_modules', 'network', 'tests', 'hybrid_2018_03_01', 'test_dns_commands', 'DnsZoneImportTest'] - # ['src', 'azure-cli-core', 'azure', 'cli', 'core', 'tests', 'test_aaz_arg', 'TestAAZArg'] - # ['src', 'azure-cli-telemetry', 'azure', 'cli', 'telemetry', 'tests', 'test_records_collection', 'TestRecordsCollection'] - class_name = testcase.attrib['classname'].split('.') - # classname="azure.cli.command_modules.network.tests" - if class_name[2] == 'command_modules': - module = class_name[3] - # classname="azure.cli.core.tests" - # classname="azure.cli.telemetry.tests" - elif class_name[2] in ['core', 'telemetry']: - module = class_name[2] - # classname="src.azure-cli.azure.cli.command_modules.network.tests" - elif class_name[4] == 'command_modules': - module = class_name[5] - # classname="src.azure-cli-core.azure.cli.core.tests" - # classname="src.azure-cli-telemetry.azure.cli.telemetry.tests" - elif class_name[1] in ['azure-cli-core', 'azure-cli-telemetry']: - module = class_name[4] - else: - logger.error(f'unexpected class name: {class_name}') - module = 'unknown' - failures = testcase.findall('failure') - if failures: - # logger.info(f"failed testcase attributes: {testcase.attrib}") - state = "Failed" - test_case = testcase.attrib['name'] - line = testcase.attrib['file'] + ':' + testcase.attrib['line'] - # only get first failure - for failure in failures: - message = failure.attrib['message'].replace('\n', '
').replace(' ', ' ') - break - for i in pipeline_result[unique_job_name]['Details'][0]['Details'][0]['Details'][0]['Details']: - if i['Module'] == module: - i['Status'] = 'Failed' - # GitHub has a comment length limit of 65535, we must ensure that the length is less than 65535. - # The azure cli bot will also add extra html characters. - # So the number of characters cannot be accurately calculated. - # Using indent=4 is just a rough estimate. - if len(json.dumps(pipeline_result, indent=4)) + len(message) > 65535: - message = 'The error message is too long, please check the pipeline log for details.' - i['Content'] = build_markdown_content(state, test_case, message, line, i['Content']) - break - else: - for i in pipeline_result[unique_job_name]['Details'][0]['Details'][0]['Details'][0]['Details']: - if i['Module'] == module: - i['Status'] = 'Succeeded' if i['Status'] != 'Failed' else 'Failed' - break +def _module_from_test_path(value): + # Accept dotted JUnit classnames and source paths, with or without a src prefix. + parts = value.replace('\\', '.').replace('/', '.').split('.') + for index in range(len(parts) - 2): + if parts[index:index + 2] == ['azure', 'cli']: + module = parts[index + 2] + if module == 'command_modules' and index + 3 < len(parts): + return parts[index + 3] + if module in ('core', 'telemetry'): + return module + for package in ('azure-cli-core', 'azure-cli-telemetry'): + if package in parts: + return package.removeprefix('azure-cli-') + return None + + +def _test_file_modules(): + # A pyproject root can shorten classnames to test_foo.TestClass. Resolve + # those from source instead of hard-coding filenames or guessing a module. + root = Path(working_directory) if working_directory else Path(__file__).resolve().parents[2] + modules = {} + for path in root.glob('src/azure-cli*/azure/cli/**/test_*.py'): + if 'tests' not in path.parts: + continue + module = _module_from_test_path(path.as_posix()) + if module: + modules.setdefault(path.stem, set()).add(module) + return modules + +def get_pipeline_result(test_result_fp, pipeline_result): + root = ET.parse(test_result_fp).getroot() + details = pipeline_result[unique_job_name]['Details'][0]['Details'][0]['Details'][0]['Details'] + test_modules = None + for testcase in root.iter('testcase'): + classname = testcase.get('classname', '') + filename = testcase.get('file', '') + module = _module_from_test_path(classname) or _module_from_test_path(filename) + if module is None: + if test_modules is None: + test_modules = _test_file_modules() + test_file = Path(filename.replace('\\', '/')).stem if filename else next( + (part for part in classname.split('.') if part.startswith('test_')), '') + matches = test_modules.get(test_file, set()) + module = next(iter(matches)) if len(matches) == 1 else 'unknown' + if module == 'unknown': + logger.warning('Unable to identify test module: classname=%r file=%r', classname, filename) + result = next((item for item in details if item['Module'] == module), None) + if result is None: + result = {'Module': module, 'Status': 'Running', 'Content': ''} + details.append(result) + failures = testcase.findall('failure') + testcase.findall('error') + if failures: + result['Status'] = 'Failed' + test_case = testcase.get('name', classname) + line = filename or classname + if testcase.get('line'): + line += ':' + testcase.get('line') + failure = failures[0] + message = (failure.get('message') or failure.text or 'See pipeline log for details.') + message = message.replace('\n', '
').replace(' ', ' ') + # Leave room for the extra HTML added by the GitHub comment bot. + if len(json.dumps(pipeline_result, indent=4)) + len(message) > 65535: + message = 'The error message is too long, please check the pipeline log for details.' + result['Content'] = build_markdown_content('Failed', test_case, message, line, result['Content']) + elif result['Status'] != 'Failed': + result['Status'] = 'Succeeded' return pipeline_result diff --git a/tools/automation/tests/test_pipeline_results.py b/tools/automation/tests/test_pipeline_results.py new file mode 100644 index 00000000000..58be847624e --- /dev/null +++ b/tools/automation/tests/test_pipeline_results.py @@ -0,0 +1,178 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Offline regression tests for CI result aggregation after package migration.""" + +import importlib.util +from pathlib import Path +import sys +import tempfile +from types import ModuleType +import unittest +from unittest import mock +import xml.etree.ElementTree as ET + + +class PipelineResultTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + script = Path(__file__).resolve().parents[3] / 'scripts/ci/automation_full_test.py' + spec = importlib.util.spec_from_file_location('automation_full_test', script) + cls.runner = importlib.util.module_from_spec(spec) + # Loading the reporting functions must not require installing azdev, + # resolving repositories, or running the scheduler. + utilities = ModuleType('azdev.utilities') + utilities.get_path_table = mock.Mock(side_effect=AssertionError('Unexpected azdev discovery')) + with mock.patch.dict(sys.modules, {'azdev': ModuleType('azdev'), 'azdev.utilities': utilities}), \ + mock.patch.object(sys, 'argv', [str(script)]): + spec.loader.exec_module(cls.runner) + cls.addClassCleanup(cls.runner.logger.removeHandler, cls.runner.ch) + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.patch = mock.patch.multiple(self.runner, working_directory=str(self.root), unique_job_name='job') + self.patch.start() + self.addCleanup(self.patch.stop) + self.reset_result() + self.add_source('azure-cli-telemetry/azure/cli/telemetry/tests/test_telemetry_note.py') + self.add_source('azure-cli-core/azure/cli/core/tests/test_aaz_arg.py') + + def reset_result(self): + self.rows = [{'Module': name, 'Status': 'Running', 'Content': ''} + for name in ('network', 'core', 'telemetry')] + self.result = {'job': {'Details': [{'Details': [{'Details': [{'Details': self.rows}]}]}]}} + + def add_source(self, relative): + path = self.root / 'src' / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + def aggregate(self, cases, single_suite=False): + root = ET.Element('testsuite' if single_suite else 'testsuites') + suite = root if single_suite else ET.SubElement(root, 'testsuite') + for attributes, outcome in cases: + case = ET.SubElement(suite, 'testcase', dict(name='test_example', **attributes)) + if outcome: + kind, message = outcome + node = ET.SubElement(case, kind) + if message is not None: + node.set('message', message) + node.text = 'Failure details' + path = self.root / 'results.xml' + ET.ElementTree(root).write(path, encoding='utf-8') + returned = self.runner.get_pipeline_result(str(path), self.result) + self.assertIs(returned, self.result) + return {row['Module']: row for row in self.rows} + + def test_short_telemetry_classname_from_migrated_package(self): + for filename in ('test_telemetry_note.py', 'src/azure-cli/test_telemetry_note.py'): + with self.subTest(filename=filename): + self.reset_result() + rows = self.aggregate([({'classname': 'test_telemetry_note.TestTelemetryNote', + 'file': filename}, None)]) + self.assertEqual(rows['telemetry']['Status'], 'Succeeded') + self.assertEqual(rows['core']['Status'], 'Running') + + def test_short_classname_without_file_uses_source_inventory(self): + rows = self.aggregate([({'classname': 'test_aaz_arg.TestAAZArg'}, None)]) + self.assertEqual(rows['core']['Status'], 'Succeeded') + + def test_short_command_module_classname_uses_source_inventory(self): + self.add_source('azure-cli/azure/cli/command_modules/network/tests/latest/test_network.py') + rows = self.aggregate([({'classname': 'test_network.TestNetwork'}, None)]) + self.assertEqual(rows['network']['Status'], 'Succeeded') + + def test_existing_qualified_names_remain_supported(self): + for classname, module in ( + ('azure.cli.command_modules.network.tests.latest.test_network.TestNetwork', 'network'), + ('src.azure-cli.azure.cli.command_modules.network.tests.hybrid_2018_03_01.test_network.TestNetwork', 'network'), + ('azure.cli.core.tests.test_aaz_arg.TestAAZArg', 'core'), + ('src.azure-cli-core.azure.cli.core.tests.test_aaz_arg.TestAAZArg', 'core'), + ('azure.cli.telemetry.tests.test_telemetry_note.TestTelemetryNote', 'telemetry'), + ('src.azure-cli-telemetry.azure.cli.telemetry.tests.test_telemetry_note.TestTelemetryNote', 'telemetry'), + ): + with self.subTest(classname=classname): + self.reset_result() + with mock.patch.object(self.runner, '_test_file_modules', + side_effect=AssertionError('Unexpected source scan')): + rows = self.aggregate([({'classname': classname}, None)]) + self.assertEqual(rows[module]['Status'], 'Succeeded') + + def test_file_path_identifies_module_without_classname(self): + for file_name in ( + 'src/azure-cli/azure/cli/command_modules/network/tests/latest/test_commands.py', + r'C:\repo\src\azure-cli\azure\cli\command_modules\network\tests\test_commands.py', + ): + with self.subTest(file_name=file_name): + self.reset_result() + rows = self.aggregate([({'file': file_name}, None)]) + self.assertEqual(rows['network']['Status'], 'Succeeded') + + def test_xunit1_failure_location_and_failed_status_are_preserved(self): + rows = self.aggregate([ + ({'classname': 'test_telemetry_note.TestTelemetryNote', 'file': 'test_telemetry_note.py', + 'line': '42'}, ('failure', 'unexpected\nvalue')), + ({'classname': 'test_telemetry_note.TestTelemetryNote'}, None), + ]) + self.assertEqual(rows['telemetry']['Status'], 'Failed') + self.assertIn('unexpected
value', rows['telemetry']['Content']) + self.assertIn('test_telemetry_note.py:42', rows['telemetry']['Content']) + + def test_failure_survives_a_later_report(self): + case = {'classname': 'test_telemetry_note.TestTelemetryNote'} + self.aggregate([(case, ('failure', 'first report failed'))]) + rows = self.aggregate([(case, None), (case, ('skipped', 'not applicable'))]) + self.assertEqual(rows['telemetry']['Status'], 'Failed') + self.assertIn('first report failed', rows['telemetry']['Content']) + + def test_oversized_failure_keeps_existing_message_limit(self): + rows = self.aggregate([({'classname': 'test_aaz_arg.TestAAZArg'}, ('failure', 'x' * 65536))]) + self.assertEqual(rows['core']['Status'], 'Failed') + self.assertIn('The error message is too long', rows['core']['Content']) + self.assertLess(len(rows['core']['Content']), 1000) + + def test_xunit2_error_without_location_or_message_is_reported(self): + rows = self.aggregate([({'classname': 'test_aaz_arg.TestAAZArg'}, ('error', None))]) + self.assertEqual(rows['core']['Status'], 'Failed') + self.assertIn('Failure details', rows['core']['Content']) + + def test_unknown_or_missing_classname_does_not_hide_failure(self): + for count, attributes in enumerate(({'classname': 'Unexpected'}, {}, {'classname': 'a.b.command_modules'}), 1): + with self.subTest(attributes=attributes): + rows = self.aggregate([(attributes, ('failure', 'cannot attribute'))]) + self.assertEqual(rows['unknown']['Status'], 'Failed') + self.assertIn('cannot attribute', rows['unknown']['Content']) + self.assertEqual(rows['unknown']['Content'].count('|Failed|'), count) + self.assertEqual(rows['core']['Status'], 'Running') + self.assertEqual(sum(row['Module'] == 'unknown' for row in self.rows), 1) + + def test_ambiguous_short_name_is_not_assigned_to_wrong_module(self): + self.add_source('azure-cli-core/azure/cli/core/tests/test_shared.py') + self.add_source('azure-cli-telemetry/azure/cli/telemetry/tests/test_shared.py') + rows = self.aggregate([({'classname': 'test_shared.TestShared'}, ('failure', 'ambiguous'))]) + self.assertEqual(rows['unknown']['Status'], 'Failed') + self.assertEqual(rows['core']['Status'], 'Running') + self.assertEqual(rows['telemetry']['Status'], 'Running') + + def test_single_suite_root_is_supported(self): + rows = self.aggregate([({'classname': 'test_telemetry_note.TestTelemetryNote'}, None)], single_suite=True) + self.assertEqual(rows['telemetry']['Status'], 'Succeeded') + + def test_reported_ci_batch_completes_after_passing_tests(self): + network = {'classname': 'azure.cli.command_modules.network.tests.latest.test_network.TestNetwork'} + telemetry = {'classname': 'test_telemetry_note.TestTelemetryNote', + 'file': 'test_telemetry_note.py'} + with mock.patch.object(self.runner, '_test_file_modules', wraps=self.runner._test_file_modules) as inventory: + rows = self.aggregate([(network, None)] * 305 + [(telemetry, None)] * 2 + + [(network, ('skipped', 'not applicable'))] * 78) + inventory.assert_called_once_with() + self.assertEqual(rows['network']['Status'], 'Succeeded') + self.assertEqual(rows['telemetry']['Status'], 'Succeeded') + + +if __name__ == '__main__': + unittest.main()