diff --git a/.azure-pipelines/templates/automation_test.yml b/.azure-pipelines/templates/automation_test.yml index dab01694208..742745a5ca6 100644 --- a/.azure-pipelines/templates/automation_test.yml +++ b/.azure-pipelines/templates/automation_test.yml @@ -27,6 +27,12 @@ parameters: displayName: Job name type: string default: '' +- name: AzdevPackage + type: string + default: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' +- name: DiffToolPackage + type: string + default: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' steps: - task: UsePythonVersion@0 @@ -36,6 +42,8 @@ steps: - template: ./azdev_setup.yml parameters: EnableCompactAAZ: true + AzdevPackage: ${{ parameters.AzdevPackage }} + DiffToolPackage: ${{ parameters.DiffToolPackage }} - bash: | set -ev diff --git a/.azure-pipelines/templates/azdev_setup.yml b/.azure-pipelines/templates/azdev_setup.yml index 95c81836dee..454998cc268 100644 --- a/.azure-pipelines/templates/azdev_setup.yml +++ b/.azure-pipelines/templates/azdev_setup.yml @@ -8,6 +8,14 @@ parameters: - name: EnableCompactAAZ type: boolean default: false + # Temporary immutable candidates until the validated packages are released. + # Keep both requirements on the same commit; overrides may also be wheel paths. + - name: AzdevPackage + type: string + default: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' + - name: DiffToolPackage + type: string + default: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' steps: - bash: | set -ev @@ -16,8 +24,12 @@ steps: chmod +x env/bin/activate . env/bin/activate - python -m pip install -U pip "setuptools<81" - pip install --upgrade "azdev==0.2.13" + python -m pip install -U pip "setuptools>=78.1.1" + packages=("$AZDEV_PACKAGE") + if [ -n "$DIFF_TOOL_PACKAGE" ]; then + packages+=("$DIFF_TOOL_PACKAGE") + fi + python -m pip install --upgrade "${packages[@]}" azdev --version if [ -z "$CLI_EXT_REPO_PATH" ]; then @@ -37,5 +49,7 @@ steps: az --version displayName: 'azdev setup' env: + AZDEV_PACKAGE: ${{ parameters.AzdevPackage }} + DIFF_TOOL_PACKAGE: ${{ parameters.DiffToolPackage }} CLI_REPO_PATH: ${{ parameters.CLIRepoPath }} CLI_EXT_REPO_PATH: ${{ parameters.CLIExtensionRepoPath }} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12e6d10f0a0..d727b2c523a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,8 +22,8 @@ /src/azure-cli/ @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad -/src/azure-cli-core/setup.py @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad -/src/azure-cli/setup.py @Azure/act-identity-squad @Azure/act-codegen-extensibility-squad +/src/azure-cli-core/pyproject.toml @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad +/src/azure-cli/pyproject.toml @Azure/act-identity-squad @Azure/act-codegen-extensibility-squad /src/azure-cli/requirements.*.txt @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad /src/azure-cli/azure/cli/command_modules/acr/ @Azure/act-observability-squad diff --git a/.github/actions/env-setup/action.yml b/.github/actions/env-setup/action.yml index b5953254f8e..029534eb5f8 100644 --- a/.github/actions/env-setup/action.yml +++ b/.github/actions/env-setup/action.yml @@ -23,13 +23,17 @@ runs: python-version: "3.12" - name: Install azdev shell: bash + # Temporary immutable candidates; neither package needs a PyPI release. + env: + AZDEV_PACKAGE: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' + DIFF_TOOL_PACKAGE: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' run: | python -m pip install --upgrade pip set -ev python -m venv env chmod +x env/bin/activate source ./env/bin/activate - pip install azdev + python -m pip install "$AZDEV_PACKAGE" "$DIFF_TOOL_PACKAGE" azdev --version cd ../ azdev setup -c azure-cli --debug diff --git a/azure-cli.pyproj b/azure-cli.pyproj index 687601ef6dd..dd3ac6195a0 100644 --- a/azure-cli.pyproj +++ b/azure-cli.pyproj @@ -75,7 +75,7 @@ - + @@ -84,11 +84,11 @@ - + - + diff --git a/azure-cli2017.pyproj b/azure-cli2017.pyproj index dc1d455ca7d..ecb6467b380 100644 --- a/azure-cli2017.pyproj +++ b/azure-cli2017.pyproj @@ -87,7 +87,7 @@ - + @@ -118,8 +118,7 @@ - - + @@ -129,7 +128,7 @@ - + @@ -963,7 +962,7 @@ - + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8584c302aa8..29c279685f2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -541,6 +541,28 @@ jobs: pythonVersion: '$(python.version)' module: 'azure-cli-core' +- job: TestPackagingMetadata + displayName: Test Packaging Metadata + timeoutInMinutes: 10 + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python310: + python.version: '3.10' + Python314: + python.version: '3.14' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + - bash: | + set -e + python -m pip install docutils requests packaging tomli + python -B -m unittest discover -s tools/automation/tests -t tools -p 'test_*.py' -v + python -B scripts/ci/check_package_versions.py + displayName: Validate source metadata and packaging automation + - job: TestTelemetry displayName: Unit Test for Telemetry timeoutInMinutes: 10 @@ -577,7 +599,7 @@ jobs: displayName: 'Use Python $(python.version)' inputs: versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel build "setuptools<81" + - bash: pip install --upgrade pip wheel build "setuptools>=78.1.1" displayName: 'Install pip and wheel' - bash: ./scripts/ci/test_profile_integration.sh displayName: 'Run Integration Test against Profiles' @@ -598,7 +620,7 @@ jobs: displayName: 'Use Python $(python.version)' inputs: versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel build "setuptools<81" + - bash: pip install --upgrade pip wheel build "setuptools>=78.1.1" displayName: 'Install pip and wheel setuptools' - bash: ./scripts/ci/test_extensions.sh displayName: 'Load extensions' diff --git a/requirements.txt b/requirements.txt index d854618ebf9..f658b6490fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ # basic -# Cap setuptools<81: newer setuptools breaks the CLI's setup.py-based builds -# (81 removes setup.py --dry-run and changes distutils command signatures; -# 82 removes pkg_resources). Pin to the last 80.x line. -setuptools>=65.5.1,<81 +# Security floor for setuptools; CLI packages use the PEP 517 build backend. +setuptools>=78.1.1 pip>=9.0.1 diff --git a/scripts/ci/automation_full_test.py b/scripts/ci/automation_full_test.py index e4dc30cae80..1ae10f55216 100644 --- a/scripts/ci/automation_full_test.py +++ b/scripts/ci/automation_full_test.py @@ -9,6 +9,7 @@ import json import logging import os +from pathlib import Path import subprocess import sys import time @@ -423,60 +424,73 @@ def build_pipeline_result(): return pipeline_result -def get_pipeline_result(test_result_fp, pipeline_result): - tree = ET.parse(test_result_fp) - root = tree.getroot() - for testsuite in root: - for testcase in testsuite: - # ['azure', 'cli', 'command_modules', 'network', 'tests', 'latest', 'test_network_commands', 'NetworkNicScenarioTest'] - # ['src', 'azure-cli', 'azure', 'cli', 'command_modules', 'network', 'tests', 'hybrid_2018_03_01', 'test_dns_commands', 'DnsZoneImportTest'] - # ['src', 'azure-cli-core', 'azure', 'cli', 'core', 'tests', 'test_aaz_arg', 'TestAAZArg'] - # ['src', 'azure-cli-telemetry', 'azure', 'cli', 'telemetry', 'tests', 'test_records_collection', 'TestRecordsCollection'] - class_name = testcase.attrib['classname'].split('.') - # classname="azure.cli.command_modules.network.tests" - if class_name[2] == 'command_modules': - module = class_name[3] - # classname="azure.cli.core.tests" - # classname="azure.cli.telemetry.tests" - elif class_name[2] in ['core', 'telemetry']: - module = class_name[2] - # classname="src.azure-cli.azure.cli.command_modules.network.tests" - elif class_name[4] == 'command_modules': - module = class_name[5] - # classname="src.azure-cli-core.azure.cli.core.tests" - # classname="src.azure-cli-telemetry.azure.cli.telemetry.tests" - elif class_name[1] in ['azure-cli-core', 'azure-cli-telemetry']: - module = class_name[4] - else: - logger.error(f'unexpected class name: {class_name}') - module = 'unknown' - failures = testcase.findall('failure') - if failures: - # logger.info(f"failed testcase attributes: {testcase.attrib}") - state = "Failed" - test_case = testcase.attrib['name'] - line = testcase.attrib['file'] + ':' + testcase.attrib['line'] - # only get first failure - for failure in failures: - message = failure.attrib['message'].replace('\n', '
').replace(' ', ' ') - break - for i in pipeline_result[unique_job_name]['Details'][0]['Details'][0]['Details'][0]['Details']: - if i['Module'] == module: - i['Status'] = 'Failed' - # GitHub has a comment length limit of 65535, we must ensure that the length is less than 65535. - # The azure cli bot will also add extra html characters. - # So the number of characters cannot be accurately calculated. - # Using indent=4 is just a rough estimate. - if len(json.dumps(pipeline_result, indent=4)) + len(message) > 65535: - message = 'The error message is too long, please check the pipeline log for details.' - i['Content'] = build_markdown_content(state, test_case, message, line, i['Content']) - break - else: - for i in pipeline_result[unique_job_name]['Details'][0]['Details'][0]['Details'][0]['Details']: - if i['Module'] == module: - i['Status'] = 'Succeeded' if i['Status'] != 'Failed' else 'Failed' - break +def _module_from_test_path(value): + # Accept dotted JUnit classnames and source paths, with or without a src prefix. + parts = value.replace('\\', '.').replace('/', '.').split('.') + for index in range(len(parts) - 2): + if parts[index:index + 2] == ['azure', 'cli']: + module = parts[index + 2] + if module == 'command_modules' and index + 3 < len(parts): + return parts[index + 3] + if module in ('core', 'telemetry'): + return module + for package in ('azure-cli-core', 'azure-cli-telemetry'): + if package in parts: + return package.removeprefix('azure-cli-') + return None + + +def _test_file_modules(): + # A pyproject root can shorten classnames to test_foo.TestClass. Resolve + # those from source instead of hard-coding filenames or guessing a module. + root = Path(working_directory) if working_directory else Path(__file__).resolve().parents[2] + modules = {} + for path in root.glob('src/azure-cli*/azure/cli/**/test_*.py'): + if 'tests' not in path.parts: + continue + module = _module_from_test_path(path.as_posix()) + if module: + modules.setdefault(path.stem, set()).add(module) + return modules + +def get_pipeline_result(test_result_fp, pipeline_result): + root = ET.parse(test_result_fp).getroot() + details = pipeline_result[unique_job_name]['Details'][0]['Details'][0]['Details'][0]['Details'] + test_modules = None + for testcase in root.iter('testcase'): + classname = testcase.get('classname', '') + filename = testcase.get('file', '') + module = _module_from_test_path(classname) or _module_from_test_path(filename) + if module is None: + if test_modules is None: + test_modules = _test_file_modules() + test_file = Path(filename.replace('\\', '/')).stem if filename else next( + (part for part in classname.split('.') if part.startswith('test_')), '') + matches = test_modules.get(test_file, set()) + module = next(iter(matches)) if len(matches) == 1 else 'unknown' + if module == 'unknown': + logger.warning('Unable to identify test module: classname=%r file=%r', classname, filename) + result = next((item for item in details if item['Module'] == module), None) + if result is None: + result = {'Module': module, 'Status': 'Running', 'Content': ''} + details.append(result) + failures = testcase.findall('failure') + testcase.findall('error') + if failures: + result['Status'] = 'Failed' + test_case = testcase.get('name', classname) + line = filename or classname + if testcase.get('line'): + line += ':' + testcase.get('line') + failure = failures[0] + message = (failure.get('message') or failure.text or 'See pipeline log for details.') + message = message.replace('\n', '
').replace(' ', ' ') + # Leave room for the extra HTML added by the GitHub comment bot. + if len(json.dumps(pipeline_result, indent=4)) + len(message) > 65535: + message = 'The error message is too long, please check the pipeline log for details.' + result['Content'] = build_markdown_content('Failed', test_case, message, line, result['Content']) + elif result['Status'] != 'Failed': + result['Status'] = 'Succeeded' return pipeline_result diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index c5a49c759f5..845a5d8b67b 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -55,13 +55,15 @@ title 'Build Azure CLI and its command modules' # provisioning the frontend is easy to miss. Install it here if it is absent rather # than failing partway through the build. python -c 'import build' 2>/dev/null || python -m pip install --disable-pip-version-check -q build +python -c 'import tomllib' 2>/dev/null || python -m pip install --disable-pip-version-check -q 'tomli>=2.0.1' +python "$script_dir/check_package_versions.py" -for setup_file in $(find src -name 'setup.py'); do - pushd $(dirname ${setup_file}) >/dev/null +for pyproject_file in $(find src -name 'pyproject.toml'); do + pushd $(dirname ${pyproject_file}) >/dev/null echo "Building module at $(pwd) ..." # --no-isolation builds against the environment prepared by the caller rather than - # provisioning a fresh one, which keeps the caller's setuptools pin in force and - # avoids requiring outbound network access inside the packaging test containers. + # provisioning a fresh one. The caller must provide the build requirements; + # this avoids fetching them inside the packaging test containers. python -m build --wheel --no-isolation --outdir $output_dir python -m build --sdist --no-isolation --outdir $sdist_dir popd >/dev/null diff --git a/scripts/ci/check_package_versions.py b/scripts/ci/check_package_versions.py new file mode 100644 index 00000000000..062011d8f5d --- /dev/null +++ b/scripts/ci/check_package_versions.py @@ -0,0 +1,32 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Validate source versions before building release or nightly distributions.""" + +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[2] +PACKAGES = ('azure-cli', 'azure-cli-core', 'azure-cli-telemetry', 'azure-cli-testsdk') +sys.path.insert(0, str(ROOT / 'tools')) + +from automation.utilities.packaging import get_package_version, tomllib # noqa: E402 + + +def main(): + versions = {} + for name in PACKAGES: + versions[name] = get_package_version(ROOT / 'src' / name) + with (ROOT / 'src/azure-cli/pyproject.toml').open('rb') as stream: + dependencies = tomllib.load(stream)['project']['dependencies'] + expected = 'azure-cli-core==' + versions['azure-cli-core'] + if expected not in dependencies or versions['azure-cli'] != versions['azure-cli-core']: + raise ValueError('CLI and core versions must match, including the dependency: ' + expected) + for name, version in versions.items(): + print('{}: {}'.format(name, version)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/ci/version.sh b/scripts/ci/version.sh index 4494e89ac9c..907ac53458c 100755 --- a/scripts/ci/version.sh +++ b/scripts/ci/version.sh @@ -32,11 +32,13 @@ for each in $(find src -name __init__.py); do fi done -for each in $(find src -name setup.py); do +for each in $(find src -name pyproject.toml); do if [ "$platform" == "Darwin" ]; then - sed -i "" "s/^VERSION = [\"']\(.*\)[\"']/VERSION = \"\1.$version\"/" $each + sed -i "" "s/^version = [\"']\(.*\)[\"']/version = \"\1.$version\"/" "$each" + sed -i "" "s/azure-cli-core==\([^\"']*\)/azure-cli-core==\1.$version/" "$each" else - sed -i "s/^VERSION = [\"']\(.*\)[\"']/VERSION = \"\1.$version\"/" $each + sed -i "s/^version = [\"']\(.*\)[\"']/version = \"\1.$version\"/" "$each" + sed -i "s/azure-cli-core==\([^\"']*\)/azure-cli-core==\1.$version/" "$each" fi done diff --git a/scripts/install_full.sh b/scripts/install_full.sh index bdda1836ef8..62e3dae3aa2 100755 --- a/scripts/install_full.sh +++ b/scripts/install_full.sh @@ -16,7 +16,7 @@ REPO_ROOT="$(dirname ${BASH_SOURCE[0]})/.." pushd ${REPO_ROOT} > /dev/null -find src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps +find src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps pip install -r ./src/azure-cli/requirements.$(python ./scripts/get-python-version.py).$(uname).txt popd > /dev/null diff --git a/scripts/live_test/CLITest.yml b/scripts/live_test/CLITest.yml index 69ff49977ab..75e8e1fd513 100644 --- a/scripts/live_test/CLITest.yml +++ b/scripts/live_test/CLITest.yml @@ -13,6 +13,11 @@ trigger: variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml +# Temporary immutable candidates, matching the shared CI setup. +- name: AzdevPackage + value: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e' +- name: DiffToolPackage + value: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool' parameters: - name: maxParallel @@ -558,7 +563,7 @@ stages: source env/bin/activate # TODO: support pytest-html >= 4.0.0 - pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures + python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures azdev setup -c azure-cli -r azure-cli-extensions @@ -1136,7 +1141,7 @@ stages: export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES # TODO: support pytest-html >= 4.0.0 - pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures + python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures azdev setup -c azure-cli -r azure-cli-extensions @@ -1712,7 +1717,7 @@ stages: source env/Scripts/activate # TODO: support pytest-html >= 4.0.0 - pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures + python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures azdev setup -c azure-cli -r azure-cli-extensions diff --git a/scripts/regression_test/regression_test.yml b/scripts/regression_test/regression_test.yml index f618b0c0dfb..7149cb978cd 100644 --- a/scripts/regression_test/regression_test.yml +++ b/scripts/regression_test/regression_test.yml @@ -46,8 +46,8 @@ jobs: echo "'PACKAGE' and 'TARGET_PACKAGE_VERSION' are required" exit 1 fi - sed -i "s/'$(PACKAGE)==.*'/'$(PACKAGE)==$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py - sed -i "s/'$(PACKAGE)~=.*'/'$(PACKAGE)~=$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py + sed -i "s/'$(PACKAGE)==.*'/'$(PACKAGE)==$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/pyproject.toml + sed -i "s/'$(PACKAGE)~=.*'/'$(PACKAGE)~=$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/pyproject.toml sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.windows.txt sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Linux.txt sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Darwin.txt diff --git a/scripts/release/debian/build.sh b/scripts/release/debian/build.sh index 141f2b79ba0..a918c1ce002 100755 --- a/scripts/release/debian/build.sh +++ b/scripts/release/debian/build.sh @@ -37,12 +37,12 @@ $PYTHON_SRC_DIR/*/configure --srcdir $PYTHON_SRC_DIR/* --prefix $WORKDIR/python_ make make install -# Cap setuptools<81: 81 removes setup.py --dry-run and changes distutils command signatures (82 removes pkg_resources); the CLI build relies on setup.py. -$WORKDIR/python_env/bin/python3 -m pip install --upgrade pip "setuptools<81" +# Keep setuptools at or above the security floor. +$WORKDIR/python_env/bin/python3 -m pip install --upgrade pip "setuptools>=78.1.1" export PATH=$PATH:$WORKDIR/python_env/bin -find ${WORKDIR}/src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip3 install --no-deps +find ${WORKDIR}/src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip3 install --no-deps pip3 install -r ${WORKDIR}/src/azure-cli/requirements.py3.$(uname).txt $WORKDIR/python_env/bin/python3 ${WORKDIR}/scripts/trim_sdk.py diff --git a/scripts/release/homebrew/docker/requirements.txt b/scripts/release/homebrew/docker/requirements.txt index 0e37f95638a..0dd9c04cdc4 100644 --- a/scripts/release/homebrew/docker/requirements.txt +++ b/scripts/release/homebrew/docker/requirements.txt @@ -1,7 +1,7 @@ -# Pin setuptools<81 to stay aligned with the CLI-wide cap: homebrew-pypi-poet -# (abandoned since 2018) uses pkg_resources, which was removed in setuptools 82+, -# and 81 changes setup.py/distutils behavior. See https://github.com/Azure/azure-cli/issues/32800 -setuptools<81 +# homebrew-pypi-poet uses pkg_resources, removed in setuptools 82. +# Keep this cap local to formula-generation tooling, with the security floor. +# See https://github.com/Azure/azure-cli/issues/32800 +setuptools>=78.1.1,<82 homebrew-pypi-poet~=0.10.0 jinja2~=3.1.6 requests>=2.20.0 diff --git a/scripts/release/homebrew/docker/run.sh b/scripts/release/homebrew/docker/run.sh index 1ab98f2da92..b19e7a42bd3 100755 --- a/scripts/release/homebrew/docker/run.sh +++ b/scripts/release/homebrew/docker/run.sh @@ -13,7 +13,7 @@ fi pip install wheel pip install -U pip pip install -r $root/requirements.txt -find /mnt/src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps +find /mnt/src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps pip install -r /mnt/src/azure-cli/requirements.py3.Darwin.txt pip list diff --git a/scripts/release/pypi/build.sh b/scripts/release/pypi/build.sh index 6857dbc9b70..96535056e63 100755 --- a/scripts/release/pypi/build.sh +++ b/scripts/release/pypi/build.sh @@ -14,14 +14,12 @@ cd $BUILD_SOURCESDIRECTORY branch=$1 echo "Branch $branch" -echo "Search setup files from `pwd`." +echo "Search pyproject.toml files from `pwd`." python --version -# Cap setuptools<81: setuptools 82 removes pkg_resources, which azure-cli still declares as a -# runtime dependency. The builds below go through the PEP 517 frontend, but --no-isolation means -# they use this setuptools too. -# `build` is that frontend. -pip install -U pip "setuptools<81" wheel build +# Provision the PEP 517 frontend and backend for --no-isolation builds, +# keeping setuptools at or above the security floor. +pip install -U pip "setuptools>=78.1.1" wheel build pip list script_dir=`cd $(dirname $BASH_SOURCE[0]); pwd` @@ -30,8 +28,11 @@ if [[ ! $branch =~ ^release ]]; then . $script_dir/../../ci/version.sh post`date -u '+%Y%m%d%H%M%S'` fi -for setup_file in $(find src -name 'setup.py' | grep -v azure-cli-testsdk); do - pushd `dirname $setup_file` +python -c 'import tomllib' 2>/dev/null || python -m pip install --disable-pip-version-check -q 'tomli>=2.0.1' +python "$script_dir/../../ci/check_package_versions.py" + +for pyproject_file in $(find src -name 'pyproject.toml' | grep -v azure-cli-testsdk); do + pushd `dirname $pyproject_file` python -m build --wheel --no-isolation --outdir $BUILD_STAGINGDIRECTORY python -m build --sdist --no-isolation --outdir $BUILD_STAGINGDIRECTORY popd diff --git a/scripts/release/rpm/test_azurelinux_in_docker.sh b/scripts/release/rpm/test_azurelinux_in_docker.sh index 35f74c72d60..ae4b2794b48 100644 --- a/scripts/release/rpm/test_azurelinux_in_docker.sh +++ b/scripts/release/rpm/test_azurelinux_in_docker.sh @@ -14,10 +14,9 @@ time az self-test time az --version cd /azure-cli/ -# Cap setuptools<81: 81 removes setup.py --dry-run and changes distutils command signatures (82 removes pkg_resources). -# scripts/ci/build.sh builds with `python -m build --no-isolation`, so this pin is the setuptools the build uses. -# `build` is the PEP 517 frontend that script invokes. -python -m pip install --upgrade "setuptools<81" build +# Provision build and setuptools for scripts/ci/build.sh's --no-isolation builds, +# keeping setuptools at or above the security floor. +python -m pip install --upgrade "setuptools>=78.1.1" build ./scripts/ci/build.sh # From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh index de1be306174..8eb556eba1e 100644 --- a/scripts/release/rpm/test_rpm_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -16,10 +16,9 @@ time az self-test time az --version cd /azure-cli/ -# Cap setuptools<81: 81 removes setup.py --dry-run and changes distutils command signatures (82 removes pkg_resources). -# scripts/ci/build.sh builds with `python -m build --no-isolation`, so this pin is the setuptools the build uses. -# `build` is the PEP 517 frontend that script invokes. -python -m pip install --upgrade pip "setuptools<81" build +# Provision build and setuptools for scripts/ci/build.sh's --no-isolation builds, +# keeping setuptools at or above the security floor. +python -m pip install --upgrade pip "setuptools>=78.1.1" build ./scripts/ci/build.sh # From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. diff --git a/src/azure-cli-core/pyproject.toml b/src/azure-cli-core/pyproject.toml new file mode 100644 index 00000000000..e63644e9ed2 --- /dev/null +++ b/src/azure-cli-core/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli-core" +version = "2.91.0" +description = "Microsoft Azure Command-Line Tools Core Module" +readme = { file = "README.rst", content-type = "text/x-rst" } +requires-python = ">=3.10.0" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + 'argcomplete~=3.5.2', + 'azure-cli-telemetry==1.1.0.*', + 'azure-core~=1.39.0', + 'azure-mgmt-core>=1.2.0,<2', + 'cryptography', + # On Linux, the distribution (Ubuntu, Debian, etc) and version are logged in telemetry + 'distro; sys_platform == "linux"', + 'humanfriendly~=10.0', + 'jmespath', + 'knack~=0.14.0', + 'microsoft-security-utilities-secret-masker~=1.0.0b4', + 'msal-extensions==1.3.1', + 'msal[broker]==1.39.0; sys_platform == "win32" or sys_platform == "darwin"', + 'msal==1.39.0; sys_platform != "win32" and sys_platform != "darwin"', + 'packaging>=20.9', + # pkginfo>=1.12.0 reads the spec-defined wheel METADATA / unpacked .dist-info + # layout produced by modern wheel/setuptools (no metadata.json). Required so + # WheelExtension.get_metadata works without the legacy wheel==0.30.0 artifact. + 'pkginfo>=1.12.0', + # psutil can't install on cygwin: https://github.com/Azure/azure-cli/issues/9399 + 'psutil>=5.9; sys_platform != "cygwin"', + 'PyJWT>=2.1.0', + 'pyopenssl>=17.1.0', # https://github.com/pyca/pyopenssl/pull/612 + 'py-deviceid', + 'requests[socks]', +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false + +[tool.setuptools.packages.find] +namespaces = false +exclude = ["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"] + +[tool.setuptools.package-data] +"azure.cli.core" = ["auth/landing_pages/*.html", "commandIndex.latest.json", "helpIndex.latest.json"] \ No newline at end of file diff --git a/src/azure-cli-core/setup.py b/src/azure-cli-core/setup.py deleted file mode 100644 index 5c1bc3668f7..00000000000 --- a/src/azure-cli-core/setup.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup, find_packages - -VERSION = "2.91.0" - -# If we have source, validate that our version numbers match -# This should prevent uploading releases with mismatched versions. -try: - with open('azure/cli/core/__init__.py', 'r', encoding='utf-8') as f: - content = f.read() -except OSError: - pass -else: - import re - import sys - - m = re.search(r'__version__\s*=\s*[\'"](.+?)[\'"]', content) - if not m: - print('Could not find __version__ in azure/cli/core/__init__.py') - sys.exit(1) - if m.group(1) != VERSION: - print('Expected __version__ = "{}"; found "{}"'.format(VERSION, m.group(1))) - sys.exit(1) - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', -] - -DEPENDENCIES = [ - 'argcomplete~=3.5.2', - 'azure-cli-telemetry==1.1.0.*', - 'azure-core~=1.39.0', - 'azure-mgmt-core>=1.2.0,<2', - 'cryptography', - # On Linux, the distribution (Ubuntu, Debian, etc) and version are logged in telemetry - 'distro; sys_platform == "linux"', - 'humanfriendly~=10.0', - 'jmespath', - 'knack~=0.14.0', - 'microsoft-security-utilities-secret-masker~=1.0.0b4', - 'msal-extensions==1.3.1', - 'msal[broker]==1.39.0; sys_platform == "win32" or sys_platform == "darwin"', - 'msal==1.39.0; sys_platform != "win32" and sys_platform != "darwin"', - 'packaging>=20.9', - # pkginfo>=1.12.0 reads the spec-defined wheel METADATA / unpacked .dist-info - # layout produced by modern wheel/setuptools (no metadata.json). Required so - # WheelExtension.get_metadata works without the legacy wheel==0.30.0 artifact. - 'pkginfo>=1.12.0', - # psutil can't install on cygwin: https://github.com/Azure/azure-cli/issues/9399 - 'psutil>=5.9; sys_platform != "cygwin"', - 'PyJWT>=2.1.0', - 'pyopenssl>=17.1.0', # https://github.com/pyca/pyopenssl/pull/612 - 'py-deviceid', - 'requests[socks]', -] - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() - -setup( - name='azure-cli-core', - version=VERSION, - description='Microsoft Azure Command-Line Tools Core Module', - long_description=README, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"]), - install_requires=DEPENDENCIES, - python_requires='>=3.10.0', - package_data={'azure.cli.core': ['auth/landing_pages/*.html', 'commandIndex.latest.json', 'helpIndex.latest.json']} -) diff --git a/src/azure-cli-telemetry/pyproject.toml b/src/azure-cli-telemetry/pyproject.toml new file mode 100644 index 00000000000..9998646086e --- /dev/null +++ b/src/azure-cli-telemetry/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli-telemetry" +version = "1.1.0" +description = "Microsoft Azure CLI Telemetry Package" +dynamic = ["readme"] +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + 'applicationinsights>=0.11.1,<0.12', + 'portalocker>=1.6,<4', +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false +packages = ["azure.cli.telemetry", "azure.cli.telemetry.components"] + +[tool.setuptools.dynamic] +readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" } \ No newline at end of file diff --git a/src/azure-cli-telemetry/setup.py b/src/azure-cli-telemetry/setup.py deleted file mode 100755 index 63a6b21cc61..00000000000 --- a/src/azure-cli-telemetry/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup - -VERSION = "1.1.0" - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', -] - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() -with open('HISTORY.rst', 'r', encoding='utf-8') as f: - HISTORY = f.read() - -setup( - name='azure-cli-telemetry', - version=VERSION, - description='Microsoft Azure CLI Telemetry Package', - long_description=README + '\n\n' + HISTORY, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - install_requires=[ - 'applicationinsights>=0.11.1,<0.12', - 'portalocker>=1.6,<4', - ], - packages=[ - 'azure.cli.telemetry', - 'azure.cli.telemetry.components' - ] -) diff --git a/src/azure-cli-testsdk/pyproject.toml b/src/azure-cli-testsdk/pyproject.toml new file mode 100644 index 00000000000..efb4889660f --- /dev/null +++ b/src/azure-cli-testsdk/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "azure-cli-testsdk" +version = "0.3.0" +description = "Microsoft Azure Command-Line Tools Test SDK Module" +dynamic = ["readme"] +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + 'ConfigArgParse>=0.12.0', + 'jmespath', + 'vcrpy>=1.10.3', + 'pytest', +] + +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false +packages = ["azure.cli.testsdk", "azure.cli.testsdk.scenario_tests"] + +[tool.setuptools.dynamic] +readme = { file = ["README.rst", "HISTORY.rst"], content-type = "text/x-rst" } \ No newline at end of file diff --git a/src/azure-cli-testsdk/setup.py b/src/azure-cli-testsdk/setup.py deleted file mode 100644 index d32fe0bebcc..00000000000 --- a/src/azure-cli-testsdk/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup - -VERSION = "0.3.0" - -CLASSIFIERS = [ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', -] - -DEPENDENCIES = [ - 'ConfigArgParse>=0.12.0', - 'jmespath', - 'vcrpy>=1.10.3', - 'pytest' -] - -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() -with open('HISTORY.rst', 'r', encoding='utf-8') as f: - HISTORY = f.read() - -setup( - name='azure-cli-testsdk', - version=VERSION, - description='Microsoft Azure Command-Line Tools Test SDK Module', - long_description=README + '\n\n' + HISTORY, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - packages=[ - 'azure.cli.testsdk', - 'azure.cli.testsdk.scenario_tests', - ], - install_requires=DEPENDENCIES -) diff --git a/src/azure-cli/setup.py b/src/azure-cli/pyproject.toml similarity index 50% rename from src/azure-cli/setup.py rename to src/azure-cli/pyproject.toml index e2c2fbf0297..b00ac47822f 100644 --- a/src/azure-cli/setup.py +++ b/src/azure-cli/pyproject.toml @@ -1,61 +1,34 @@ -#!/usr/bin/env python +[build-system] +requires = ["setuptools>=78.1.1"] +build-backend = "setuptools.build_meta" -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from codecs import open -from setuptools import setup, find_packages -import sys - -try: - from azure_cli_bdist_wheel import cmdclass -except ImportError: - import logging - - logging.warning("Wheel is not available, disabling bdist_wheel hook") - cmdclass = {} - -VERSION = "2.91.0" -# If we have source, validate that our version numbers match -# This should prevent uploading releases with mismatched versions. -try: - with open('azure/cli/__main__.py', 'r', encoding='utf-8') as f: - content = f.read() -except OSError: - pass -else: - import re - - m = re.search(r'__version__\s*=\s*[\'"](.+?)[\'"]', content) - if not m: - print('Could not find __version__ in azure/cli/__main__.py') - sys.exit(1) - if m.group(1) != VERSION: - print('Expected __version__ = "{}"; found "{}"'.format(VERSION, m.group(1))) - sys.exit(1) - -CLASSIFIERS = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - 'Programming Language :: Python :: 3.14', - 'License :: OSI Approved :: MIT License', +[project] +name = "azure-cli" +version = "2.91.0" +description = "Microsoft Azure Command-Line Tools" +readme = { file = "README.rst", content-type = "text/x-rst" } +requires-python = ">=3.10.0" +license = "MIT" +license-files = ["LICENSE.txt"] +authors = [{ name = "Microsoft Corporation", email = "azpycli@microsoft.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] - -DEPENDENCIES = [ +dependencies = [ "antlr4-python3-runtime~=4.13.1", 'azure-appconfiguration==1.10.0b1', 'azure-ai-projects~=1.0.0', 'azure-batch==15.0.0b1', - 'azure-cli-core=={}'.format(VERSION), + 'azure-cli-core==2.91.0', 'azure-cosmos~=3.0,>=3.0.2', 'azure-data-tables==12.4.0', 'azure-datalake-store~=1.0.1', @@ -75,7 +48,7 @@ 'azure-mgmt-billing==6.0.0', 'azure-mgmt-botservice~=2.0.0b3', 'azure-mgmt-cdn==12.0.0', - 'azure-mgmt-cognitiveservices==15.0.0b4', # 15.0.0b4 version supports compute operations which is in PrPr, pls don't remove it without consulting our team + 'azure-mgmt-cognitiveservices==15.0.0b4', # Supports compute operations in private preview; consult the team before removing. 'azure-mgmt-compute~=34.1.0', 'azure-mgmt-containerinstance==10.2.0b1', 'azure-mgmt-containerregistry==15.1.0b3', @@ -154,71 +127,60 @@ 'PyNaCl~=1.6.2', 'scp~=0.13.2', 'semver~=3.0', - 'six>=1.10.0', # six is still used by countless extensions + 'six>=1.10.0', # six is still used by countless extensions 'sshtunnel~=0.1.4', # Even though knack already depends on tabulate, profile module directly uses it for interactive subscription # selection 'tabulate', 'urllib3', 'websocket-client~=1.8.0', - 'xmltodict~=0.12' + 'xmltodict~=0.12', ] -with open('README.rst', 'r', encoding='utf-8') as f: - README = f.read() +[project.urls] +Homepage = "https://github.com/Azure/azure-cli" + +[tool.setuptools] +include-package-data = false +zip-safe = false +script-files = ["az", "az.completion.sh", "az.bat", "azps.ps1"] + +[tool.setuptools.packages.find] +namespaces = false +exclude = ["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"] + +[tool.setuptools.package-data] +"azure.cli.command_modules.acr" = ["*.json"] +"azure.cli.command_modules.botservice" = ["*.json", "*.config"] +"azure.cli.command_modules.monitor.operations" = ["autoscale-parameters-template.json"] +"azure.cli.command_modules.servicefabric" = [ + "template/windows/template.json", + "template/windows/parameter.json", + "template/linux/template.json", + "template/linux/parameter.json", + "template/service/template.json", + "template/service/parameter.json", +] +"azure.cli.command_modules.appservice" = [ + "resources/WebappRuntimeStacks.json", + "resources/GenerateRandomAppNames.json", +] +"azure.cli.command_modules.rdbms" = [ + "*.json", + "randomname/adjectives.txt", + "randomname/nouns.txt", + "templates/mysql_githubaction_template.yaml", +] +"azure.cli.command_modules.mysql" = [ + "random/adjectives.txt", + "random/nouns.txt", +] +"azure.cli.command_modules.postgresql" = [ + "*.json", + "randomname/adjectives.txt", + "randomname/nouns.txt", + "templates/postgresql_githubaction_template.yaml", +] -setup( - name='azure-cli', - version=VERSION, - description='Microsoft Azure Command-Line Tools', - long_description=README, - license='MIT', - author='Microsoft Corporation', - author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', - zip_safe=False, - classifiers=CLASSIFIERS, - scripts=[ - 'az', - 'az.completion.sh', - 'az.bat', - 'azps.ps1' - ], - packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", "azure", "azure.cli"]), - install_requires=DEPENDENCIES, - python_requires='>=3.10.0', - package_data={ - 'azure.cli.command_modules.acr': ['*.json'], - 'azure.cli.command_modules.botservice': ['*.json', '*.config'], - 'azure.cli.command_modules.monitor.operations': ['autoscale-parameters-template.json'], - 'azure.cli.command_modules.servicefabric': [ - 'template/windows/template.json', - 'template/windows/parameter.json', - 'template/linux/template.json', - 'template/linux/parameter.json', - 'template/service/template.json', - 'template/service/parameter.json' - ], - 'azure.cli.command_modules.appservice': [ - 'resources/WebappRuntimeStacks.json', - 'resources/GenerateRandomAppNames.json' - ], - 'azure.cli.command_modules.rdbms': [ - '*.json', - 'randomname/adjectives.txt', - 'randomname/nouns.txt', - 'templates/mysql_githubaction_template.yaml' - ], - 'azure.cli.command_modules.mysql': [ - 'random/adjectives.txt', - 'random/nouns.txt' - ], - 'azure.cli.command_modules.postgresql': [ - '*.json', - 'randomname/adjectives.txt', - 'randomname/nouns.txt', - 'templates/postgresql_githubaction_template.yaml' - ] - }, - cmdclass=cmdclass -) +[tool.setuptools.cmdclass] +build_py = "azure_cli_bdist_wheel.azure_cli_build_py" \ No newline at end of file diff --git a/src/azure-cli/requirements.py3.Darwin.txt b/src/azure-cli/requirements.py3.Darwin.txt index ab8c676c17a..8d3645cefda 100644 --- a/src/azure-cli/requirements.py3.Darwin.txt +++ b/src/azure-cli/requirements.py3.Darwin.txt @@ -115,7 +115,7 @@ msal-extensions==1.3.1 msal==1.39.0 msrest==0.7.1 oauthlib==3.2.2 -packaging==25.0 +packaging==26.3 paramiko==5.0.0 pbr==7.0.3 pkginfo==1.12.1.2 diff --git a/src/azure-cli/requirements.py3.Linux.txt b/src/azure-cli/requirements.py3.Linux.txt index 9c0d9958690..06279030793 100644 --- a/src/azure-cli/requirements.py3.Linux.txt +++ b/src/azure-cli/requirements.py3.Linux.txt @@ -116,7 +116,7 @@ msal-extensions==1.3.1 msal==1.39.0 msrest==0.7.1 oauthlib==3.2.2 -packaging==25.0 +packaging==26.3 paramiko==5.0.0 pbr==7.0.3 pkginfo==1.12.1.2 diff --git a/src/azure-cli/requirements.py3.windows.txt b/src/azure-cli/requirements.py3.windows.txt index b600e67c8ce..adf6386f3d2 100644 --- a/src/azure-cli/requirements.py3.windows.txt +++ b/src/azure-cli/requirements.py3.windows.txt @@ -115,7 +115,7 @@ msal-extensions==1.3.1 msal[broker]==1.39.0 msrest==0.7.1 oauthlib==3.2.2 -packaging==25.0 +packaging==26.3 paramiko==5.0.0 pbr==7.0.3 pkginfo==1.12.1.2 diff --git a/tools/automation/tests/test_packaging_migration.py b/tools/automation/tests/test_packaging_migration.py new file mode 100644 index 00000000000..f7dc9cd1079 --- /dev/null +++ b/tools/automation/tests/test_packaging_migration.py @@ -0,0 +1,321 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Offline packaging regression tests; subprocesses and PyPI access are mocked.""" + +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from automation.utilities import packaging, path as package_paths +from automation.tests import verify_package_versions as versions +from automation.tests import verify_readme_history as readme +from automation.verify import default_modules + + +RUNTIME_PATHS = { + 'azure-cli': 'azure/cli/__main__.py', + 'azure-cli-core': 'azure/cli/core/__init__.py', +} + + +class PackageFixture(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='cli packaging tests ') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + + def write(self, relative_path, text): + target = self.root / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding='utf-8') + return target + + def package(self, name, version='1.2.3', parent='src'): + folder = Path(parent) / name + self.write(folder / 'pyproject.toml', + '[project]\nname = "{}"\nversion = "{}"\n'.format(name, version)) + if name in RUNTIME_PATHS: + self.write(folder / RUNTIME_PATHS[name], + 'raise AssertionError("Runtime must not be imported")\n' + '__version__ = "{}"\n'.format(version)) + return self.root / folder + + +class PackageVersionTests(PackageFixture): + def test_static_metadata_for_all_four_distributions_without_execution(self): + with mock.patch.object(packaging.subprocess, 'check_output') as execute: + for name in ('azure-cli', 'azure-cli-core', 'azure-cli-telemetry', 'azure-cli-testsdk'): + with self.subTest(name=name): + folder = self.package(name) + self.write(folder / 'setup.py', 'raise AssertionError("Do not execute")\n') + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + execute.assert_not_called() + + def test_metadata_read_uses_toml_not_text_matching(self): + folder = self.package('azure-cli-telemetry') + self.write(folder / 'pyproject.toml', + '# version = "wrong"\n[tool.example]\nversion = "also-wrong"\n' + "[project]\nname = 'azure-cli-telemetry'\nversion = '1.2.3' # comment\n") + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + + def test_tomli_fallback_when_tomllib_is_unavailable(self): + spec = importlib.util.spec_from_file_location('packaging_fallback', packaging.__file__) + module = importlib.util.module_from_spec(spec) + # Exercise the import fallback without requiring both parsers installed. + with mock.patch.dict(sys.modules, {'tomllib': None, 'tomli': packaging.tomllib}): + spec.loader.exec_module(module) + self.assertEqual(module.get_package_version(self.package('azure-cli-testsdk')), '1.2.3') + + def test_invalid_modern_metadata_never_falls_back_to_setup(self): + folder = self.package('azure-cli-testsdk') + self.write(folder / 'setup.py', '# legacy metadata must not hide invalid TOML\n') + cases = ( + ('[project\n', 'Invalid TOML'), + ('[project]\nname = "azure-cli-testsdk"\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\nversion = 123\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\nversion = ""\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\ndynamic = ["version"]\n', 'static'), + ('[project]\nname = "azure-cli-testsdk"\nversion = "1.2.3"\n' + 'dynamic = ["version"]\n', 'static'), + ('[project]\nversion = "1.2.3"\n', 'name'), + ) + with mock.patch.object(packaging.subprocess, 'check_output') as execute: + for content, message in cases: + with self.subTest(content=content): + self.write(folder / 'pyproject.toml', content) + with self.assertRaisesRegex(ValueError, message): + packaging.get_package_version(folder) + execute.assert_not_called() + + def test_runtime_mismatch_reports_both_versions_and_paths(self): + for name, runtime_file in RUNTIME_PATHS.items(): + with self.subTest(name=name): + folder = self.package(name) + self.write(folder / runtime_file, '__version__ = "9.8.7"\n') + with self.assertRaises(ValueError) as caught: + packaging.get_package_version(folder) + for detail in (name, '1.2.3', '9.8.7', str(folder / 'pyproject.toml'), + str(folder / runtime_file)): + self.assertIn(detail, str(caught.exception)) + + def test_runtime_must_be_one_top_level_literal(self): + folder = self.package('azure-cli-core') + for content in ( + '# __version__ = "1.2.3"\n', + '__version__ = get_version()\n', + '__version__ = 123\n', + '__version__: str\n', + '__version__ = "1.2.3"\n__version__ = "1.2.3"\n', + 'def nested():\n __version__ = "1.2.3"\n'): + with self.subTest(content=content): + self.write(folder / RUNTIME_PATHS['azure-cli-core'], content) + with self.assertRaisesRegex(ValueError, 'top-level string literal __version__'): + packaging.get_package_version(folder) + + def test_annotated_runtime_literal_is_supported(self): + folder = self.package('azure-cli-core') + self.write(folder / RUNTIME_PATHS['azure-cli-core'], '__version__: str = "1.2.3"\n') + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + + def test_missing_or_invalid_runtime_source_is_not_silently_skipped(self): + folder = self.package('azure-cli') + runtime_file = folder / RUNTIME_PATHS['azure-cli'] + runtime_file.unlink() + with self.assertRaisesRegex(ValueError, 'Unable to read runtime __version__'): + packaging.get_package_version(folder) + self.write(runtime_file, '__version__ = (\n') + with self.assertRaisesRegex(ValueError, 'Unable to read runtime __version__'): + packaging.get_package_version(folder) + + def test_legacy_fallback_uses_current_interpreter_and_package_directory(self): + folder = self.root / 'legacy package' + self.write(folder / 'setup.py', '# historical setup\n') + with mock.patch.object(packaging.subprocess, 'check_output', return_value='1.2.3\n') as execute: + self.assertEqual(packaging.get_package_version(folder), '1.2.3') + execute.assert_called_once_with( + [sys.executable, 'setup.py', '--version'], cwd=str(folder), universal_newlines=True) + + def test_missing_package_has_useful_error(self): + with self.assertRaisesRegex(FileNotFoundError, 'No pyproject.toml or setup.py'): + packaging.get_package_version(self.root) + + def test_historical_base_repo_comparison_retains_legacy_fallback(self): + current = self.root / 'current' + base = self.root / 'historical' + relative = Path('src') / 'azure-cli' + self.write(base / relative / 'setup.py', '# historical setup\n') + with mock.patch.object(versions, 'get_repo_root', return_value=str(current)), \ + mock.patch.object(packaging.subprocess, 'check_output', return_value='1.2.3\n') as execute: + self.assertTrue(versions.version_in_base_repo( + str(base), str(current / relative), 'azure-cli', '1.2.3')) + execute.assert_called_once_with( + [sys.executable, 'setup.py', '--version'], cwd=str(base / relative), universal_newlines=True) + + def test_version_validator_checks_runtime_before_contacting_pypi(self): + folder = self.package('azure-cli') + self.write(folder / RUNTIME_PATHS['azure-cli'], '__version__ = "9.8.7"\n') + with mock.patch.object(versions, 'is_available_on_pypi') as pypi: + with self.assertRaisesRegex(ValueError, 'Version mismatch'): + versions.check_package_version('azure-cli', str(folder)) + pypi.assert_not_called() + + +class PackageDiscoveryTests(PackageFixture): + def test_core_discovery_includes_four_pyprojects_legacy_and_deduplicates(self): + names = ('azure-cli', 'azure-cli-core', 'azure-cli-telemetry', 'azure-cli-testsdk') + expected = [(name, str(self.package(name))) for name in names] + self.write('src/azure-cli/setup.py', '# transitional package\n') + self.write('src/legacy/setup.py', '# legacy\n') + expected.append(('legacy', str(self.root / 'src' / 'legacy'))) + self.write('src/not-a-package/README.rst', 'Not a package\n') + (self.root / 'src' / 'not-a-package' / 'pyproject.toml').mkdir() + with mock.patch.object(package_paths, 'get_repo_root', return_value=str(self.root)): + self.assertEqual(sorted(package_paths.get_core_modules_paths()), sorted(expected)) + + def test_command_discovery_preserves_prefix_option_and_combined_paths(self): + core = self.package('azure-cli-core') + command = self.package('azure-cli-modern', parent='src/command_modules') + self.write(command / 'setup.py', '# transitional package\n') + legacy = self.root / 'src' / 'command_modules' / 'azure-cli-legacy' + self.write(legacy / 'setup.py', '# legacy\n') + self.package('not-a-command', parent='src/command_modules') + with mock.patch.object(package_paths, 'get_repo_root', return_value=str(self.root)): + self.assertEqual(list(package_paths.get_command_modules_paths()), + [('legacy', str(legacy)), ('modern', str(command))]) + prefixed = [('azure-cli-legacy', str(legacy)), ('azure-cli-modern', str(command))] + self.assertEqual(list(package_paths.get_command_modules_paths(include_prefix=True)), prefixed) + self.assertEqual(package_paths.get_all_module_paths(), [('azure-cli-core', str(core))] + prefixed) + + def test_default_modules_points_to_main_pyproject(self): + self.assertEqual(Path(default_modules.AZURE_CLI_PYPROJECT), + Path(default_modules.AZURE_CLI_PATH) / 'pyproject.toml') + + +class ReleaseVersionTests(PackageFixture): + def setUp(self): + super().setUp() + script = Path(__file__).resolve().parents[3] / 'scripts/ci/check_package_versions.py' + spec = importlib.util.spec_from_file_location('check_package_versions', script) + self.checker = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.checker) + self.checker.ROOT = self.root + for name in self.checker.PACKAGES: + self.package(name) + with (self.root / 'src/azure-cli/pyproject.toml').open('a', encoding='utf-8') as stream: + stream.write('dependencies = ["azure-cli-core==1.2.3"]\n') + + def test_matching_versions_pass(self): + self.checker.main() + + def test_missing_distribution_fails(self): + (self.root / 'src/azure-cli-testsdk/pyproject.toml').unlink() + with self.assertRaises(FileNotFoundError): + self.checker.main() + + def test_mismatched_core_dependency_fails(self): + self.package('azure-cli-core', version='1.2.4') + with self.assertRaisesRegex(ValueError, 'including the dependency'): + self.checker.main() + + def test_nightly_versions_and_dependency_are_synchronized(self): + for name in self.checker.PACKAGES: + self.package(name, version='1.2.3.dev20260918000000') + with (self.root / 'src/azure-cli/pyproject.toml').open('a', encoding='utf-8') as stream: + stream.write('dependencies = ["azure-cli-core==1.2.3.dev20260918000000"]\n') + self.checker.main() + + +class ReadmeValidationTests(PackageFixture): + def history(self, folder, version='1.2.3'): + self.write(folder / 'HISTORY.rst', + 'Release History\n===============\n\n{}\n{}\n\n* Changes.\n\n' + '0.0.1\n-----\n\n* Initial release.\n'.format( + version, '-' * len(version))) + + def test_history_uses_static_source_version(self): + folder = self.package('azure-cli-telemetry') + self.history(folder) + with mock.patch.object(packaging.subprocess, 'check_output') as execute: + self.assertTrue(readme.check_history_headings(str(folder))) + self.history(folder, '9.8.7') + self.assertFalse(readme.check_history_headings(str(folder))) + execute.assert_not_called() + + def test_history_retains_runtime_mismatch_validation(self): + folder = self.package('azure-cli-core') + self.history(folder) + self.write(folder / RUNTIME_PATHS['azure-cli-core'], '__version__ = "9.8.7"\n') + with self.assertRaisesRegex(ValueError, 'Version mismatch'): + readme.check_history_headings(str(folder)) + + def test_build_and_twine_use_clean_temporary_copy_and_leave_source_untouched(self): + folder = self.package('azure-cli-telemetry') + self.history(folder) + self.write(folder / 'README.rst', 'Original readme\n') + self.write(folder / 'azure/cli/telemetry/__init__.py', '# package\n') + ignored = ('tests/recordings/large.yaml', 'build/stale.txt', 'dist/stale.tar.gz', + '__pycache__/stale.pyc', 'old.egg-info/PKG-INFO', '.venv/large.txt') + for relative in ignored: + self.write(folder / relative, 'stale\n') + before = {p.relative_to(folder): p.read_bytes() for p in folder.rglob('*') if p.is_file()} + commands = [] + working_dirs = [] + + def execute(command, **kwargs): + commands.append(command) + source = Path(kwargs['cwd']) + working_dirs.append(source) + self.assertNotEqual(source, folder) + self.assertTrue((source / 'pyproject.toml').is_file()) + if command[2] == 'build': + self.assertEqual(command[:5], [sys.executable, '-m', 'build', '--sdist', '--outdir']) + self.assertTrue((source / 'azure/cli/telemetry/__init__.py').is_file()) + for relative in ignored: + self.assertFalse((source / relative).exists()) + (source / 'README.rst').write_text('Backend changed the copy\n', encoding='utf-8') + output = Path(command[5]) + output.mkdir() + (output / 'azure_cli_telemetry-1.2.3.tar.gz').write_bytes(b'mocked sdist') + else: + self.assertEqual(command[:5], [sys.executable, '-m', 'twine', 'check', '--strict']) + self.assertEqual(len(command), 6) + self.assertTrue(Path(command[5]).is_file()) + + with mock.patch.object(readme.subprocess, 'check_call', side_effect=execute): + self.assertTrue(readme.check_readme_render(str(folder))) + self.assertEqual(len(commands), 2) + self.assertEqual(working_dirs[0], working_dirs[1]) + self.assertFalse(working_dirs[0].parent.exists()) + after = {p.relative_to(folder): p.read_bytes() for p in folder.rglob('*') if p.is_file()} + self.assertEqual(before, after) + + def test_failed_build_missing_sdist_and_twine_failure_are_reported_and_cleaned(self): + folder = self.package('azure-cli-testsdk') + for failure in ('build', 'missing-sdist', 'twine'): + with self.subTest(failure=failure): + working_dirs = [] + + def execute(command, **kwargs): + working_dirs.append(Path(kwargs['cwd'])) + if command[2] == failure: + raise subprocess.CalledProcessError(1, command) + if command[2] == 'build' and failure != 'missing-sdist': + output = Path(command[5]) + output.mkdir() + (output / 'package-1.2.3.tar.gz').write_bytes(b'mocked sdist') + + with mock.patch.object(readme.subprocess, 'check_call', side_effect=execute) as call: + self.assertFalse(readme._check_sdist_render(str(folder))) + self.assertEqual(call.call_count, 2 if failure == 'twine' else 1) + self.assertFalse(working_dirs[0].parent.exists()) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tools/automation/tests/test_pipeline_results.py b/tools/automation/tests/test_pipeline_results.py new file mode 100644 index 00000000000..58be847624e --- /dev/null +++ b/tools/automation/tests/test_pipeline_results.py @@ -0,0 +1,178 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Offline regression tests for CI result aggregation after package migration.""" + +import importlib.util +from pathlib import Path +import sys +import tempfile +from types import ModuleType +import unittest +from unittest import mock +import xml.etree.ElementTree as ET + + +class PipelineResultTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + script = Path(__file__).resolve().parents[3] / 'scripts/ci/automation_full_test.py' + spec = importlib.util.spec_from_file_location('automation_full_test', script) + cls.runner = importlib.util.module_from_spec(spec) + # Loading the reporting functions must not require installing azdev, + # resolving repositories, or running the scheduler. + utilities = ModuleType('azdev.utilities') + utilities.get_path_table = mock.Mock(side_effect=AssertionError('Unexpected azdev discovery')) + with mock.patch.dict(sys.modules, {'azdev': ModuleType('azdev'), 'azdev.utilities': utilities}), \ + mock.patch.object(sys, 'argv', [str(script)]): + spec.loader.exec_module(cls.runner) + cls.addClassCleanup(cls.runner.logger.removeHandler, cls.runner.ch) + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.patch = mock.patch.multiple(self.runner, working_directory=str(self.root), unique_job_name='job') + self.patch.start() + self.addCleanup(self.patch.stop) + self.reset_result() + self.add_source('azure-cli-telemetry/azure/cli/telemetry/tests/test_telemetry_note.py') + self.add_source('azure-cli-core/azure/cli/core/tests/test_aaz_arg.py') + + def reset_result(self): + self.rows = [{'Module': name, 'Status': 'Running', 'Content': ''} + for name in ('network', 'core', 'telemetry')] + self.result = {'job': {'Details': [{'Details': [{'Details': [{'Details': self.rows}]}]}]}} + + def add_source(self, relative): + path = self.root / 'src' / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + def aggregate(self, cases, single_suite=False): + root = ET.Element('testsuite' if single_suite else 'testsuites') + suite = root if single_suite else ET.SubElement(root, 'testsuite') + for attributes, outcome in cases: + case = ET.SubElement(suite, 'testcase', dict(name='test_example', **attributes)) + if outcome: + kind, message = outcome + node = ET.SubElement(case, kind) + if message is not None: + node.set('message', message) + node.text = 'Failure details' + path = self.root / 'results.xml' + ET.ElementTree(root).write(path, encoding='utf-8') + returned = self.runner.get_pipeline_result(str(path), self.result) + self.assertIs(returned, self.result) + return {row['Module']: row for row in self.rows} + + def test_short_telemetry_classname_from_migrated_package(self): + for filename in ('test_telemetry_note.py', 'src/azure-cli/test_telemetry_note.py'): + with self.subTest(filename=filename): + self.reset_result() + rows = self.aggregate([({'classname': 'test_telemetry_note.TestTelemetryNote', + 'file': filename}, None)]) + self.assertEqual(rows['telemetry']['Status'], 'Succeeded') + self.assertEqual(rows['core']['Status'], 'Running') + + def test_short_classname_without_file_uses_source_inventory(self): + rows = self.aggregate([({'classname': 'test_aaz_arg.TestAAZArg'}, None)]) + self.assertEqual(rows['core']['Status'], 'Succeeded') + + def test_short_command_module_classname_uses_source_inventory(self): + self.add_source('azure-cli/azure/cli/command_modules/network/tests/latest/test_network.py') + rows = self.aggregate([({'classname': 'test_network.TestNetwork'}, None)]) + self.assertEqual(rows['network']['Status'], 'Succeeded') + + def test_existing_qualified_names_remain_supported(self): + for classname, module in ( + ('azure.cli.command_modules.network.tests.latest.test_network.TestNetwork', 'network'), + ('src.azure-cli.azure.cli.command_modules.network.tests.hybrid_2018_03_01.test_network.TestNetwork', 'network'), + ('azure.cli.core.tests.test_aaz_arg.TestAAZArg', 'core'), + ('src.azure-cli-core.azure.cli.core.tests.test_aaz_arg.TestAAZArg', 'core'), + ('azure.cli.telemetry.tests.test_telemetry_note.TestTelemetryNote', 'telemetry'), + ('src.azure-cli-telemetry.azure.cli.telemetry.tests.test_telemetry_note.TestTelemetryNote', 'telemetry'), + ): + with self.subTest(classname=classname): + self.reset_result() + with mock.patch.object(self.runner, '_test_file_modules', + side_effect=AssertionError('Unexpected source scan')): + rows = self.aggregate([({'classname': classname}, None)]) + self.assertEqual(rows[module]['Status'], 'Succeeded') + + def test_file_path_identifies_module_without_classname(self): + for file_name in ( + 'src/azure-cli/azure/cli/command_modules/network/tests/latest/test_commands.py', + r'C:\repo\src\azure-cli\azure\cli\command_modules\network\tests\test_commands.py', + ): + with self.subTest(file_name=file_name): + self.reset_result() + rows = self.aggregate([({'file': file_name}, None)]) + self.assertEqual(rows['network']['Status'], 'Succeeded') + + def test_xunit1_failure_location_and_failed_status_are_preserved(self): + rows = self.aggregate([ + ({'classname': 'test_telemetry_note.TestTelemetryNote', 'file': 'test_telemetry_note.py', + 'line': '42'}, ('failure', 'unexpected\nvalue')), + ({'classname': 'test_telemetry_note.TestTelemetryNote'}, None), + ]) + self.assertEqual(rows['telemetry']['Status'], 'Failed') + self.assertIn('unexpected
value', rows['telemetry']['Content']) + self.assertIn('test_telemetry_note.py:42', rows['telemetry']['Content']) + + def test_failure_survives_a_later_report(self): + case = {'classname': 'test_telemetry_note.TestTelemetryNote'} + self.aggregate([(case, ('failure', 'first report failed'))]) + rows = self.aggregate([(case, None), (case, ('skipped', 'not applicable'))]) + self.assertEqual(rows['telemetry']['Status'], 'Failed') + self.assertIn('first report failed', rows['telemetry']['Content']) + + def test_oversized_failure_keeps_existing_message_limit(self): + rows = self.aggregate([({'classname': 'test_aaz_arg.TestAAZArg'}, ('failure', 'x' * 65536))]) + self.assertEqual(rows['core']['Status'], 'Failed') + self.assertIn('The error message is too long', rows['core']['Content']) + self.assertLess(len(rows['core']['Content']), 1000) + + def test_xunit2_error_without_location_or_message_is_reported(self): + rows = self.aggregate([({'classname': 'test_aaz_arg.TestAAZArg'}, ('error', None))]) + self.assertEqual(rows['core']['Status'], 'Failed') + self.assertIn('Failure details', rows['core']['Content']) + + def test_unknown_or_missing_classname_does_not_hide_failure(self): + for count, attributes in enumerate(({'classname': 'Unexpected'}, {}, {'classname': 'a.b.command_modules'}), 1): + with self.subTest(attributes=attributes): + rows = self.aggregate([(attributes, ('failure', 'cannot attribute'))]) + self.assertEqual(rows['unknown']['Status'], 'Failed') + self.assertIn('cannot attribute', rows['unknown']['Content']) + self.assertEqual(rows['unknown']['Content'].count('|Failed|'), count) + self.assertEqual(rows['core']['Status'], 'Running') + self.assertEqual(sum(row['Module'] == 'unknown' for row in self.rows), 1) + + def test_ambiguous_short_name_is_not_assigned_to_wrong_module(self): + self.add_source('azure-cli-core/azure/cli/core/tests/test_shared.py') + self.add_source('azure-cli-telemetry/azure/cli/telemetry/tests/test_shared.py') + rows = self.aggregate([({'classname': 'test_shared.TestShared'}, ('failure', 'ambiguous'))]) + self.assertEqual(rows['unknown']['Status'], 'Failed') + self.assertEqual(rows['core']['Status'], 'Running') + self.assertEqual(rows['telemetry']['Status'], 'Running') + + def test_single_suite_root_is_supported(self): + rows = self.aggregate([({'classname': 'test_telemetry_note.TestTelemetryNote'}, None)], single_suite=True) + self.assertEqual(rows['telemetry']['Status'], 'Succeeded') + + def test_reported_ci_batch_completes_after_passing_tests(self): + network = {'classname': 'azure.cli.command_modules.network.tests.latest.test_network.TestNetwork'} + telemetry = {'classname': 'test_telemetry_note.TestTelemetryNote', + 'file': 'test_telemetry_note.py'} + with mock.patch.object(self.runner, '_test_file_modules', wraps=self.runner._test_file_modules) as inventory: + rows = self.aggregate([(network, None)] * 305 + [(telemetry, None)] * 2 + + [(network, ('skipped', 'not applicable'))] * 78) + inventory.assert_called_once_with() + self.assertEqual(rows['network']['Status'], 'Succeeded') + self.assertEqual(rows['telemetry']['Status'], 'Succeeded') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/automation/tests/verify_package_versions.py b/tools/automation/tests/verify_package_versions.py index caf9cdd066a..d63bd720625 100644 --- a/tools/automation/tests/verify_package_versions.py +++ b/tools/automation/tests/verify_package_versions.py @@ -9,11 +9,10 @@ import subprocess from ..utilities.path import get_all_module_paths, get_repo_root +from ..utilities.packaging import get_package_version from ..utilities.display import print_heading from ..utilities.pypi import is_available_on_pypi -SETUP_PY_NAME = 'setup.py' - def is_unreleased_version(mod_name, mod_version): if is_available_on_pypi(mod_name, mod_version): @@ -67,8 +66,7 @@ def version_in_base_repo(base_repo, mod_path, mod_name, mod_version): def _get_mod_version(mod_path): - return subprocess.check_output(['python', 'setup.py', '--version'], cwd=mod_path, - universal_newlines=True).strip() + return get_package_version(mod_path) def check_package_version(mod_name, mod_path, base_repo=None, base_tag=None): diff --git a/tools/automation/tests/verify_readme_history.py b/tools/automation/tests/verify_readme_history.py index c18ce39d88a..ed30fe71082 100644 --- a/tools/automation/tests/verify_readme_history.py +++ b/tools/automation/tests/verify_readme_history.py @@ -9,9 +9,13 @@ import sys import argparse import subprocess +from pathlib import Path +import shutil +import tempfile from docutils import core, io from ..utilities.path import get_all_module_paths +from ..utilities.packaging import get_package_version from ..utilities.display import print_heading HISTORY_NAME = 'HISTORY.rst' @@ -63,19 +67,39 @@ def check_history_headings(mod_path): print("Unable to get versions from {}. Check formatting. e.g. there should be a new line after the 'Release History' heading.".format(history_path)) return False first_version_history = all_versions[0] - actual_version = subprocess.check_output('python setup.py --version'.split(), cwd=mod_path, universal_newlines=True) - actual_version = actual_version.strip() + actual_version = get_package_version(mod_path) if first_version_history != actual_version: - print("The topmost version in {} does not match version {} defined in setup.py.".format(history_path, actual_version)) + print("The topmost version in {} does not match source package version {}.".format( + history_path, actual_version)) return False return True +def _check_sdist_render(mod_path): + # Build only this package, not the repository. Omit tests/recordings and stale + # build metadata: they are unnecessary for README validation and can be huge. + # A real copy (not hard links) keeps backend writes away from the source tree. + with tempfile.TemporaryDirectory(prefix='cli-readme-') as temp_dir: + source_dir = Path(temp_dir) / 'source' + dist_dir = Path(temp_dir) / 'dist' + shutil.copytree(mod_path, source_dir, ignore=shutil.ignore_patterns( + '.git', '.venv', 'venv', 'env', '__pycache__', '*.pyc', '*.pyo', + '*.egg-info', '*.dist-info', '.pytest_cache', 'build', 'dist', 'tests', 'recordings')) + if not exec_command([sys.executable, '-m', 'build', '--sdist', '--outdir', str(dist_dir)], + cwd=str(source_dir)): + return False + artifacts = sorted(str(path) for path in dist_dir.glob('*.tar.gz')) + if not artifacts: + print('No sdist produced for {}'.format(mod_path), file=sys.stderr) + return False + return exec_command([sys.executable, '-m', 'twine', 'check', '--strict'] + artifacts, + cwd=str(source_dir)) + + def check_readme_render(mod_path): - checks = [] - checks.append(exec_command('python setup.py check -r -s', cwd=mod_path)) - checks.append(check_history_headings(mod_path)) - return all(checks) + history_ok = check_history_headings(mod_path) + render_ok = _check_sdist_render(mod_path) + return history_ok and render_ok def verify_all(): @@ -93,7 +117,7 @@ def verify_all(): print('The following modules have invalid README/HISTORYs:') print('\n'.join(failed_mods)) print('See above for the full warning/errors') - print('note: Line numbers in the errors map to the long_description of your setup.py.') + print('note: Rendering error line numbers refer to the built package description.') sys.exit(1) else: print('Verified READMEs of all modules successfully.', file=sys.stderr) @@ -106,7 +130,7 @@ def verify_one(mod_name): if not res: print_heading('Error whilst verifying README/HISTORY of {}!'.format(mod_name)) print('See above for the full warning/errors.') - print('note: Line numbers in the errors map to the long_description of your setup.py.') + print('note: Rendering error line numbers refer to the built package description.') sys.exit(1) diff --git a/tools/automation/utilities/packaging.py b/tools/automation/utilities/packaging.py new file mode 100644 index 00000000000..abbf3f05b7e --- /dev/null +++ b/tools/automation/utilities/packaging.py @@ -0,0 +1,92 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +"""Read source package versions without importing the CLI or invoking a build.""" + +import ast +from pathlib import Path +import subprocess +import sys +import tokenize + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib + + +_RUNTIME_VERSION_FILES = { + 'azure-cli': 'azure/cli/__main__.py', + 'azure-cli-core': 'azure/cli/core/__init__.py', +} + + +def _check_runtime_version(package_path, name, version): + relative_path = _RUNTIME_VERSION_FILES.get(name) + if relative_path is None: + return + + runtime_path = package_path / relative_path + try: + with tokenize.open(runtime_path) as source: + tree = ast.parse(source.read(), filename=str(runtime_path)) + except (OSError, SyntaxError) as error: + raise ValueError('Unable to read runtime __version__ for {} from {}: {}'.format( + name, runtime_path, error)) from error + + values = [] + for statement in tree.body: + if isinstance(statement, ast.Assign): + targets = statement.targets + elif isinstance(statement, ast.AnnAssign): + targets = [statement.target] + else: + continue + if any(isinstance(target, ast.Name) and target.id == '__version__' for target in targets): + values.append(statement.value) + + if (len(values) != 1 or not isinstance(values[0], ast.Constant) + or not isinstance(values[0].value, str)): + raise ValueError('Expected one top-level string literal __version__ in {}'.format(runtime_path)) + runtime_version = values[0].value + if runtime_version != version: + raise ValueError( + 'Version mismatch for {}: {} [project].version is {!r}, but {} __version__ is {!r}. ' + 'Update both source versions together.'.format( + name, package_path / 'pyproject.toml', version, runtime_path, runtime_version)) + + +def get_package_version(package_path): + """Prefer static PEP 621 metadata; retain setup.py for historical checkouts. + + Invalid modern metadata must fail rather than silently falling back to setup.py. + The main/core runtime literals are checked here so every source-version consumer + retains the validation previously performed by their setup scripts. + """ + package_path = Path(package_path) + pyproject_path = package_path / 'pyproject.toml' + if pyproject_path.is_file(): + try: + with pyproject_path.open('rb') as source: + metadata = tomllib.load(source) + except ValueError as error: + raise ValueError('Invalid TOML in {}: {}'.format(pyproject_path, error)) from error + project = metadata.get('project', {}) + name = project.get('name') + version = project.get('version') + if not isinstance(name, str) or not name.strip(): + raise ValueError('Expected a non-empty [project].name in {}'.format(pyproject_path)) + if (not isinstance(version, str) or not version.strip() + or 'version' in project.get('dynamic', [])): + raise ValueError('Expected a static, non-empty [project].version in {}'.format(pyproject_path)) + _check_runtime_version(package_path, name, version) + return version + + setup_path = package_path / 'setup.py' + if not setup_path.is_file(): + raise FileNotFoundError('No pyproject.toml or setup.py found in {}'.format(package_path)) + return subprocess.check_output( + [sys.executable, 'setup.py', '--version'], cwd=str(package_path), + universal_newlines=True).strip() \ No newline at end of file diff --git a/tools/automation/utilities/path.py b/tools/automation/utilities/path.py index a951b401c45..f7790688746 100644 --- a/tools/automation/utilities/path.py +++ b/tools/automation/utilities/path.py @@ -44,9 +44,8 @@ def get_extensions_paths(include_prefix=False): def get_command_modules_paths(include_prefix=False): - glob_pattern = os.path.normcase('/src/command_modules/{}*/setup.py'.format(COMMAND_MODULE_PREFIX)) - for path in glob.glob(get_repo_root() + glob_pattern): - folder = os.path.dirname(path) + folder_pattern = os.path.join(get_repo_root(), 'src', 'command_modules', COMMAND_MODULE_PREFIX + '*') + for folder in _get_package_paths(folder_pattern): name = os.path.basename(folder) if not include_prefix: name = name[len(COMMAND_MODULE_PREFIX):] @@ -68,8 +67,17 @@ def get_core_modules_paths_with_tests(profile): def get_core_modules_paths(): - for path in glob.glob(get_repo_root() + os.path.normcase('/src/*/setup.py')): - yield os.path.basename(os.path.dirname(path)), os.path.dirname(path) + for folder in _get_package_paths(os.path.join(get_repo_root(), 'src', '*')): + yield os.path.basename(folder), folder + + +def _get_package_paths(folder_pattern): + """Discover modern and legacy packages once, including transitional trees.""" + folders = set() + for filename in ('pyproject.toml', 'setup.py'): + folders.update(os.path.dirname(path) for path in glob.glob(os.path.join(folder_pattern, filename)) + if os.path.isfile(path)) + yield from sorted(folders) def get_module_paths_with_tests(modules, profile): diff --git a/tools/automation/verify/default_modules.py b/tools/automation/verify/default_modules.py index 56d8dd56d32..0473d138778 100644 --- a/tools/automation/verify/default_modules.py +++ b/tools/automation/verify/default_modules.py @@ -13,7 +13,7 @@ from automation.utilities.display import print_heading AZURE_CLI_PATH = os.path.join(get_repo_root(), 'src', 'azure-cli') -AZURE_CLI_SETUP_PY = os.path.join(AZURE_CLI_PATH, 'setup.py') +AZURE_CLI_PYPROJECT = os.path.join(AZURE_CLI_PATH, 'pyproject.toml') def get_cli_dependencies(build_folder): @@ -39,13 +39,13 @@ def verify_default_modules(args): cli_deps = get_cli_dependencies(args.build_folder) all_command_modules = get_command_modules_paths(include_prefix=True) if not cli_deps: - print('Unable to get the CLI dependencies for {}'.format(AZURE_CLI_SETUP_PY), file=sys.stderr) + print('Unable to get the CLI dependencies for {}'.format(AZURE_CLI_PYPROJECT), file=sys.stderr) sys.exit(1) for modname, _ in all_command_modules: if modname not in cli_deps: - errors_list.append("{} is not included to be installed by default! Modify {}.".format(modname, AZURE_CLI_SETUP_PY)) + errors_list.append("{} is not included to be installed by default! Modify {}.".format(modname, AZURE_CLI_PYPROJECT)) if errors_list: - print_heading('Errors whilst verifying default modules list in {}!'.format(AZURE_CLI_SETUP_PY)) + print_heading('Errors whilst verifying default modules list in {}!'.format(AZURE_CLI_PYPROJECT)) print('\n'.join(errors_list), file=sys.stderr) sys.exit(1) else: diff --git a/tools/setup.py b/tools/setup.py index 356f5b8c94b..29273d9915d 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -21,6 +21,9 @@ # Until https://gitlab.com/pycqa/flake8/issues/415 is resolved, pin version of pycodestyle DEPENDENCIES = [ + 'build>=1.2.2', + 'twine>=6.1.0', + 'tomli>=2.0.1; python_version < "3.11"', 'coverage>=4.2', 'flake8==3.5.0', 'pycodestyle==2.3.1',