diff --git a/doc/dev/packaging.md b/doc/dev/packaging.md index 3e6a8ee8031d..125f34b79c13 100644 --- a/doc/dev/packaging.md +++ b/doc/dev/packaging.md @@ -1,6 +1,6 @@ # Azure packaging -[comment]: # ( cspell:ignore myservice ) +[comment]: # ( cspell:ignore myservice azuremyservice msvc MSRUSTUP ) > **Note:** This document covers legacy packaging using `setup.py`. New packages should use `pyproject.toml` instead. See the [`sdk/template/azure-template`](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/template/azure-template) for the current package template. @@ -145,6 +145,64 @@ Since the package is Python 3 only, do NOT make this wheel universal. This usual - wheel file name suffix is `py3-none-any`, and NOT `py2.py3-none-any`. - sdist must contain a `azure/__init__.py` file that declares `azure` as a namespace package using the `pkgutil` syntax +# Native wheel builds in CI + +`sdk_build` uses `cibuildwheel` when a package declares setuptools `ext_modules` +or a `[tool.cibuildwheel]` table in `pyproject.toml`. The latter supports +non-setuptools backends, including Maturin/PyO3. The package owns its wheel +matrix, backend requirements, and native build hooks. + +For these packages, `sdk_build_package --package_type sdist` builds only the +source distribution with build isolation, which installs the backend declared +in `[build-system].requires` without invoking `cibuildwheel`. + +Set `signBinaries: true` on the package's artifact entry in the service `ci.yml` +to enable Windows and macOS wheel builds and the existing binary-signing release +path. The platform resolver uses this artifact property, not package names. + +Packages using the Microsoft Rust toolchain can opt in through +`archetype-sdk-client.yml`: + +```yaml +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-client.yml + parameters: + ServiceDirectory: myservice + InstallMsRustToolchain: true + MsRustWorkingDirectory: $(Build.SourcesDirectory)/sdk/myservice/azure-myservice + MsRustToolchainFeed: https://pkgs.dev.azure.com/azure-sdk/_packaging//nuget/v3/index.json + MsRustAdditionalTargets: aarch64-pc-windows-msvc + Artifacts: + - name: azure-myservice + safeName: azuremyservice + signBinaries: true +``` + +Replace `` with an approved organization-local feed that +provides the internal Rust toolchain. No prototype test feed is selected by +default. The working directory must contain `rust-toolchain.toml`. +The installer refuses to overwrite an existing Cargo configuration in its +`CARGO_HOME`; provide a clean build environment. +Rust installation is disabled by default and applies only to wheel-build jobs +that target an artifact with `signBinaries: true`, including on Linux. +Opted-in jobs have a 240-minute timeout and a 210-minute package-build timeout; +other jobs keep their existing limits. + +Linux wheels build inside manylinux containers. The package's `before-all` hook +must install the internal toolchain and configure Cargo source replacement +inside each container. The pipeline forwards `PIP_INDEX_URL`, +`MSRUSTUP_ACCESS_TOKEN`, `MSRUSTUP_FEED_URL`, and the +`CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_*` index, token, and credential-provider +variables. Credentials stay in the build environment, not checked-in files. +On Windows, `PYTHON_ARM64_LIB_DIR` points to the pre-provisioned ARM64 Python +import libraries; a package can use it as `PYO3_CROSS_LIB_DIR` for cross-compilation. + +This opt-in does not provision Rust for source-installing test, documentation, +or analysis jobs. Prepare those jobs separately before enabling a Rust package, +and keep real source-distribution testing. Packages with a static +`[project].version` also need version synchronization with `_version.py` for +nightly builds. No package is activated by adding this shared support. + # I already have a package that supports Python 2, can I get short version on how to udpate to Python 3 only? - Remove "universal" from setup.cfg, or completely remove the file if it was the only option diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 1e0254e4cbc8..1beadd69cec0 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -61,10 +61,26 @@ parameters: - name: EnvVars type: object default: {} + - name: InstallMsRustToolchain + type: boolean + default: false + - name: MsRustWorkingDirectory + type: string + default: '' + - name: MsRustToolchainFeed + type: string + default: '' + - name: MsRustAdditionalTargets + type: string + default: '' jobs: - job: 'Build_Linux' - timeoutInMinutes: 90 + # Native multi-target builds need more time, especially ARM64 under Linux emulation. + ${{ if parameters.InstallMsRustToolchain }}: + timeoutInMinutes: 240 + ${{ else }}: + timeoutInMinutes: 90 pool: name: $(LINUXPOOL) @@ -88,9 +104,16 @@ jobs: ArtifactSuffix: linux BuildTargetingString: ${{ parameters.BuildTargetingString }} ExcludePaths: ${{parameters.ExcludePaths}} + InstallMsRustToolchain: ${{ parameters.InstallMsRustToolchain }} + MsRustWorkingDirectory: ${{ parameters.MsRustWorkingDirectory }} + MsRustToolchainFeed: ${{ parameters.MsRustToolchainFeed }} + MsRustAdditionalTargets: ${{ parameters.MsRustAdditionalTargets }} - job: 'Build_Windows' - timeoutInMinutes: 90 + ${{ if parameters.InstallMsRustToolchain }}: + timeoutInMinutes: 240 + ${{ else }}: + timeoutInMinutes: 90 pool: name: $(WINDOWSPOOL) @@ -107,9 +130,16 @@ jobs: ArtifactSuffix: windows BuildTargetingString: ${{ parameters.BuildTargetingString }} ExcludePaths: ${{parameters.ExcludePaths}} + InstallMsRustToolchain: ${{ parameters.InstallMsRustToolchain }} + MsRustWorkingDirectory: ${{ parameters.MsRustWorkingDirectory }} + MsRustToolchainFeed: ${{ parameters.MsRustToolchainFeed }} + MsRustAdditionalTargets: ${{ parameters.MsRustAdditionalTargets }} - job: 'Build_MacOS' - timeoutInMinutes: 90 + ${{ if parameters.InstallMsRustToolchain }}: + timeoutInMinutes: 240 + ${{ else }}: + timeoutInMinutes: 90 pool: name: $(MACPOOL) @@ -133,6 +163,10 @@ jobs: ArtifactSuffix: mac BuildTargetingString: ${{ parameters.BuildTargetingString }} ExcludePaths: ${{parameters.ExcludePaths}} + InstallMsRustToolchain: ${{ parameters.InstallMsRustToolchain }} + MsRustWorkingDirectory: ${{ parameters.MsRustWorkingDirectory }} + MsRustToolchainFeed: ${{ parameters.MsRustToolchainFeed }} + MsRustAdditionalTargets: ${{ parameters.MsRustAdditionalTargets }} - job: 'Build_Extended' displayName: Build Extended diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 28b8ded76654..63f1a8a263ac 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -87,6 +87,22 @@ parameters: - name: EnvVars type: object default: {} + # Rust setup applies only to wheel jobs, not source-installing test or documentation jobs. + - name: InstallMsRustToolchain + type: boolean + default: false + # Package directory containing rust-toolchain.toml. + - name: MsRustWorkingDirectory + type: string + default: '' + # Required when Rust is enabled; no shared toolchain feed is selected by default. + - name: MsRustToolchainFeed + type: string + default: '' + # Space-separated Windows target triples; keep aligned with the package's wheel recipe. + - name: MsRustAdditionalTargets + type: string + default: '' extends: template: /eng/pipelines/templates/stages/1es-redirect.yml @@ -132,6 +148,10 @@ extends: BuildDocs: ${{ parameters.BuildDocs }} DevFeedName: ${{ parameters.DevFeedName }} EnvVars: ${{ parameters.EnvVars }} + InstallMsRustToolchain: ${{ parameters.InstallMsRustToolchain }} + MsRustWorkingDirectory: ${{ parameters.MsRustWorkingDirectory }} + MsRustToolchainFeed: ${{ parameters.MsRustToolchainFeed }} + MsRustAdditionalTargets: ${{ parameters.MsRustAdditionalTargets }} MatrixConfigs: - ${{ each config in parameters.MatrixConfigs }}: - ${{ config }} diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 982426287a10..f764bc0cc17a 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -23,6 +23,18 @@ parameters: - name: ExcludePaths type: object default: [] + - name: InstallMsRustToolchain + type: boolean + default: false + - name: MsRustWorkingDirectory + type: string + default: '' + - name: MsRustToolchainFeed + type: string + default: '' + - name: MsRustAdditionalTargets + type: string + default: '' steps: - ${{ if endsWith(variables['Build.Repository.Name'], '-pr') }}: @@ -118,6 +130,15 @@ steps: displayName: 'Install QEMU Dependencies' condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + - ${{ if parameters.InstallMsRustToolchain }}: + - template: /eng/pipelines/templates/steps/install-msrust-toolchain.yml + parameters: + ToolchainFeed: ${{ parameters.MsRustToolchainFeed }} + WorkingDirectory: ${{ parameters.MsRustWorkingDirectory }} + ${{ if eq(parameters.ArtifactSuffix, 'windows') }}: + AdditionalTargets: ${{ parameters.MsRustAdditionalTargets }} + Condition: and(succeeded(), eq(variables['ENABLE_EXTENSION_BUILD'], 'true')) + # Windows: cibuildwheel fetches CPython from api.nuget.org, which is blocked. # Pre-provision the interpreters from the Azure DevOps NuGet feed into the # cibuildwheel cache so it skips the blocked download. @@ -144,26 +165,59 @@ steps: exit 1 } Write-Host "##vso[task.setvariable variable=CIBW_CACHE_PATH]$cache" + + if ($env:INSTALL_MS_RUST_TOOLCHAIN -eq 'true') { + # Packages use this path as PYO3_CROSS_LIB_DIR when cross-compiling Windows ARM64 wheels. + $arm64Libs = Join-Path $nugetCpython "pythonarm64.$version\tools\libs" + if (-not (Test-Path $arm64Libs)) { + throw "Expected ARM64 Python import libraries at $arm64Libs" + } + Write-Host "##vso[task.setvariable variable=PYTHON_ARM64_LIB_DIR]$arm64Libs" + } displayName: 'Pre-provision CPython for cibuildwheel' condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['ENABLE_EXTENSION_BUILD'], 'true')) + env: + INSTALL_MS_RUST_TOOLCHAIN: ${{ format('{0}', parameters.InstallMsRustToolchain) }} # Linux: forward PIP_INDEX_URL into the build container so pip fetches build # dependencies from the Azure DevOps feed instead of PyPI. Using # CIBW_ENVIRONMENT_PASS_LINUX (not CIBW_ENVIRONMENT) preserves the package's # own [tool.cibuildwheel].environment (CFLAGS) setting. + # This replaces, rather than merges with, the package's environment-pass list. + # Include every variable required by its Linux build hooks. - pwsh: | - Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]PIP_INDEX_URL" - displayName: 'Forward PIP_INDEX_URL to cibuildwheel' + $passVariables = @('PIP_INDEX_URL') + if ($env:RUST_ENVIRONMENT_PASS) { + $passVariables += $env:RUST_ENVIRONMENT_PASS.Split(' ') + } + Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]$($passVariables -join ' ')" + displayName: 'Forward build environment to cibuildwheel' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + env: + ${{ if parameters.InstallMsRustToolchain }}: + RUST_ENVIRONMENT_PASS: MSRUSTUP_ACCESS_TOKEN MSRUSTUP_FEED_URL CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_INDEX CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER - pwsh: | which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive --service="${{ parameters.ServiceDirectory }}" displayName: 'Generate Packages' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - timeoutInMinutes: 80 + ${{ if parameters.InstallMsRustToolchain }}: + timeoutInMinutes: 210 + ${{ else }}: + timeoutInMinutes: 80 env: CIBW_BUILD_VERBOSITY: 3 + # The platform resolver selects DEBUG to stream native build output. + LOGLEVEL: $(SDK_BUILD_LOGLEVEL) + ${{ if parameters.InstallMsRustToolchain }}: + # Map secret tokens explicitly for container feed access. Package hooks can read them; + # enforce source-trust and credential permissions through external ADO controls. + MSRUSTUP_ACCESS_TOKEN: $(System.AccessToken) + MSRUSTUP_FEED_URL: $(MSRUSTUP_FEED_URL) + # CargoAuthenticate exports the registry token and credential-provider variables. + CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN: $(CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN) + CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER: $(CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_CREDENTIAL_PROVIDER) - pwsh: | which python diff --git a/eng/pipelines/templates/steps/install-msrust-toolchain.yml b/eng/pipelines/templates/steps/install-msrust-toolchain.yml new file mode 100644 index 000000000000..dbff0541e538 --- /dev/null +++ b/eng/pipelines/templates/steps/install-msrust-toolchain.yml @@ -0,0 +1,98 @@ +parameters: + # Use an organization-local mirror; the job token cannot authenticate directly to DevDiv. + # See https://aka.ms/rustinado. + - name: ToolchainFeed + type: string + default: '' + # Directory containing rust-toolchain.toml, which selects the compiler version. + - name: WorkingDirectory + type: string + default: '' + - name: AdditionalTargets + type: string + default: '' + # Mirror crates.io for network-isolated builds; ~force-auth makes Cargo authenticate upstream requests. + - name: CratesIoFeed + type: string + default: 'sparse+https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-rust-public~force-auth/Cargo/index/' + - name: Condition + type: string + default: succeeded() + +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + if (-not $env:RUST_TOOLCHAIN_FEED) { + throw 'An organization-local Rust toolchain feed must be configured.' + } + if (-not $env:RUST_WORKING_DIRECTORY) { + throw 'The directory containing rust-toolchain.toml must be configured.' + } + if (-not (Test-Path (Join-Path $env:RUST_WORKING_DIRECTORY 'rust-toolchain.toml') -PathType Leaf)) { + throw "No rust-toolchain.toml found in $env:RUST_WORKING_DIRECTORY" + } + displayName: 'Validate Rust build configuration' + condition: ${{ parameters.Condition }} + env: + RUST_TOOLCHAIN_FEED: ${{ parameters.ToolchainFeed }} + RUST_WORKING_DIRECTORY: ${{ parameters.WorkingDirectory }} + + # Installs the host toolchain; Linux build containers bootstrap their own. + - task: RustInstaller@1 + displayName: 'Install internal Rust toolchain' + condition: ${{ parameters.Condition }} + inputs: + toolchainFeed: ${{ parameters.ToolchainFeed }} + workingDirectory: ${{ parameters.WorkingDirectory }} + ${{ if ne(parameters.AdditionalTargets, '') }}: + additionalTargets: ${{ parameters.AdditionalTargets }} + + # Configure Cargo after RustInstaller defines CARGO_HOME. + - pwsh: | + $ErrorActionPreference = 'Stop' + Write-Host "##vso[task.setvariable variable=MSRUSTUP_FEED_URL]$env:RUST_TOOLCHAIN_FEED" + $cargoHome = $env:CARGO_HOME + if (-not $cargoHome) { + $cargoHome = [System.IO.Path]::Combine($HOME, '.cargo') + } + New-Item -ItemType Directory -Force -Path $cargoHome | Out-Null + $configPath = Join-Path $cargoHome 'config.toml' + if (Test-Path $configPath) { + throw "Refusing to overwrite an existing Cargo configuration: $configPath" + } + + # Keep the registry name aligned with the forwarded CARGO_REGISTRIES_* variables. + # Use the Git CLI so dependency fetches honor the agent's Git configuration. + @" + [registries] + azure-sdk-for-rust-public = { index = "$env:CARGO_FEED_URL" } + + [source.crates-io] + replace-with = "azure-sdk-for-rust-public" + + [net] + git-fetch-with-cli = true + "@ | Set-Content -Path $configPath -Encoding utf8 + + Write-Host "##vso[task.setvariable variable=CargoConfigPath]$configPath" + # Linux containers cannot see this config; the registry environment variable does not replace sources. + # Package hooks must write their own Cargo source replacement. + Write-Host "##vso[task.setvariable variable=CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_INDEX]$env:CARGO_FEED_URL" + displayName: 'Configure Rust build feeds' + condition: ${{ parameters.Condition }} + env: + RUST_TOOLCHAIN_FEED: ${{ parameters.ToolchainFeed }} + CARGO_FEED_URL: ${{ parameters.CratesIoFeed }} + + - task: NuGetAuthenticate@1 + displayName: 'Acquire an Azure Artifacts token' + condition: ${{ parameters.Condition }} + + # CargoAuthenticate reads SYSTEM_ACCESSTOKEN; use the token acquired by NuGetAuthenticate. + - task: CargoAuthenticate@0 + displayName: 'Authenticate cargo to the Azure Artifacts feed' + condition: ${{ parameters.Condition }} + inputs: + configFile: $(CargoConfigPath) + env: + SYSTEM_ACCESSTOKEN: $(VSS_NUGET_ACCESSTOKEN) diff --git a/eng/pipelines/templates/steps/resolve-build-platforms.yml b/eng/pipelines/templates/steps/resolve-build-platforms.yml index 2de46faf2585..5955fe572274 100644 --- a/eng/pipelines/templates/steps/resolve-build-platforms.yml +++ b/eng/pipelines/templates/steps/resolve-build-platforms.yml @@ -16,6 +16,7 @@ steps: # variable carries only package names and the decision below needs an artifact property. - pwsh: | $binaryPackages = @() + $logLevel = "INFO" foreach ($packageInfoPath in (Get-ChildItem -Recurse -Force "${{ parameters.PackagePropertiesFolder }}/*.json")) { $packageInfo = Get-Content -Raw -Path $packageInfoPath.FullName | ConvertFrom-Json @@ -28,8 +29,12 @@ steps: if ($binaryPackages) { Write-Host "Targeting package(s) with compiled binaries ($($binaryPackages -join ', ')), enabling extension build." Write-Host "##vso[task.setvariable variable=ENABLE_EXTENSION_BUILD]true" + # DEBUG streams long native builds instead of buffering output until a failure. + $logLevel = "DEBUG" } else { Write-Host "No targeted package declares compiled binaries, building on linux only." } + # Always define the variable used by the build step's LOGLEVEL mapping. + Write-Host "##vso[task.setvariable variable=SDK_BUILD_LOGLEVEL]$logLevel" displayName: Check extension package presence diff --git a/eng/tools/azure-sdk-tools/ci_tools/build.py b/eng/tools/azure-sdk-tools/ci_tools/build.py index 44177c20c026..5d35e53a7263 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/build.py +++ b/eng/tools/azure-sdk-tools/ci_tools/build.py @@ -223,7 +223,7 @@ def create_package( """ Builds a wheel and/or sdist file given a setup.py, pyproject.toml, or directory containing either. - For packages with compiled extensions (ext_modules): + For packages with compiled extensions (ext_modules or a [tool.cibuildwheel] table): - setup.py: uses cibuildwheel to build platform-specific wheels - pyproject.toml: uses cibuildwheel to build platform-specific wheels (respects [tool.cibuildwheel] config) @@ -237,19 +237,23 @@ def create_package( setup_parsed = ParsedSetup.from_path(setup_directory_or_file) should_log_build_output = logger.getEffectiveLevel() <= logging.DEBUG + # Backends such as Maturin compile extensions without declaring setuptools ext_modules. + # Honor their cibuildwheel matrix instead of building only a host wheel. + is_compiled = bool(setup_parsed.ext_modules) or setup_parsed.uses_cibuildwheel if setup_parsed.is_pyproject: # when building with pyproject, check if package has compiled extensions - if enable_wheel and setup_parsed.ext_modules: - # Use cibuildwheel for compiled extensions (respects [tool.cibuildwheel] config) - run_logged( - [sys.executable, "-m", "cibuildwheel", "--output-dir", dist], - cwd=setup_parsed.folder, - check=True, - should_stream_to_console=should_log_build_output, - ) + if is_compiled: + if enable_wheel: + # Use cibuildwheel for compiled extensions (respects [tool.cibuildwheel] config) + run_logged( + [sys.executable, "-m", "cibuildwheel", "--output-dir", dist], + cwd=setup_parsed.folder, + check=True, + should_stream_to_console=should_log_build_output, + ) if enable_sdist: - # Build sdist separately with python -m build + # Build isolation installs package-specific backends such as Maturin. run_logged( [sys.executable, "-m", "build", "-s", "-o", dist], cwd=setup_parsed.folder, @@ -289,7 +293,7 @@ def create_package( ) else: if enable_wheel: - if setup_parsed.ext_modules: + if is_compiled: run_logged( [sys.executable, "-m", "cibuildwheel", "--output-dir", dist], cwd=setup_parsed.folder, diff --git a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 2830ee15ae70..48c7619b7f7c 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -308,6 +308,7 @@ def __init__( self.is_pyproject = self.setup_filename.endswith(".toml") self.folder = os.path.dirname(self.setup_filename) + self.uses_cibuildwheel: bool = has_cibuildwheel_config(self.folder) @classmethod def from_path(cls, parse_directory_or_file: str): @@ -828,6 +829,20 @@ def parse_setup( return result +def has_cibuildwheel_config(folder: str) -> bool: + """Return whether the package opts into cibuildwheel, including non-setuptools backends.""" + pyproject_filename = os.path.join(folder, "pyproject.toml") + if not os.path.exists(pyproject_filename): + return False + + config = get_value_from_dict(get_pyproject_dict(pyproject_filename), "tool.cibuildwheel") + if config is None: + return False + if not isinstance(config, dict): + raise ValueError(f"Expected a [tool.cibuildwheel] table in {pyproject_filename}") + return True + + def get_pyproject_dict(pyproject_file: str) -> Dict[str, Any]: """ Given a pyproject.toml file, returns a dictionary of a target section. Defaults to `project` section. diff --git a/eng/tools/azure-sdk-tools/tests/test_build_interactions.py b/eng/tools/azure-sdk-tools/tests/test_build_interactions.py index 7bc1546390e8..950e9fae9141 100644 --- a/eng/tools/azure-sdk-tools/tests/test_build_interactions.py +++ b/eng/tools/azure-sdk-tools/tests/test_build_interactions.py @@ -1,6 +1,10 @@ import os, tempfile, shutil +import subprocess +import sys -from ci_tools.build import discover_targeted_packages, build_packages, build +import pytest + +from ci_tools.build import discover_targeted_packages, build_packages, build, create_package repo_root = os.path.join(os.path.dirname(__file__), "..", "..", "..", "..") integration_folder = os.path.join(os.path.dirname(__file__), "integration") @@ -38,3 +42,95 @@ def test_venv_helpers_importable(): from ci_tools.functions import get_venv_call as f_get_venv_call assert f_get_venv_call is get_venv_call + + +@pytest.fixture +def build_commands(monkeypatch): + calls = [] + monkeypatch.setattr("ci_tools.build.run_logged", lambda command, **kwargs: calls.append((command, kwargs))) + monkeypatch.setattr("ci_tools.build.get_pip_list_output", lambda executable: {}) + return calls + + +@pytest.mark.parametrize( + "scenario,cibuildwheel_config,compiled", + [ + ("pyproject_project_def", "", False), + ("pyproject_project_def_with_extension", "", True), + ("setup_py_project_def", "", False), + ("pyproject_project_def", '\n[tool.cibuildwheel]\nbuild = "cp310-*"\n', True), + ("setup_py_project_def", '\n[tool.cibuildwheel]\nbuild = "cp310-*"\n', True), + ], +) +@pytest.mark.parametrize("enable_wheel,enable_sdist", [(True, False), (True, True), (False, True)]) +def test_create_package_build_routing( + tmp_path, build_commands, scenario, cibuildwheel_config, compiled, enable_wheel, enable_sdist +): + package = tmp_path / "package" + shutil.copytree(os.path.join(integration_folder, "scenarios", scenario), package) + if cibuildwheel_config: + pyproject = package / "pyproject.toml" + contents = pyproject.read_text() if pyproject.exists() else "" + contents = contents.replace("setuptools.build_meta", "maturin") + pyproject.write_text(contents + cibuildwheel_config) + + destination = tmp_path / "dist" + create_package(str(package), str(destination), enable_wheel=enable_wheel, enable_sdist=enable_sdist) + + commands = [command for command, _ in build_commands] + cibuildwheel_command = [sys.executable, "-m", "cibuildwheel", "--output-dir", str(destination)] + assert (cibuildwheel_command in commands) is (compiled and enable_wheel) + assert all(options["cwd"] == str(package) for _, options in build_commands) + assert all(options["check"] for command, options in build_commands if "pip" not in command) + if compiled and scenario.startswith("pyproject"): + assert commands == ([cibuildwheel_command] if enable_wheel else []) + ( + [[sys.executable, "-m", "build", "-s", "-o", str(destination)]] if enable_sdist else [] + ) + elif not compiled and scenario.startswith("pyproject"): + build_flags = "-nsw" if enable_wheel and enable_sdist else "-nw" if enable_wheel else "-ns" + assert commands[-1] == [ + sys.executable, + "-m", + "build", + build_flags, + "-o", + str(destination), + ] + else: + expected_commands = [] + if enable_wheel: + expected_commands.append( + cibuildwheel_command + if compiled + else [sys.executable, "setup.py", "bdist_wheel", "-d", str(destination)] + ) + if enable_sdist: + expected_commands.append([sys.executable, "setup.py", "sdist", "-d", str(destination)]) + assert commands == expected_commands + + +def test_sdist_only_does_not_invoke_cibuildwheel(tmp_path, build_commands): + package = tmp_path / "package" + shutil.copytree(os.path.join(integration_folder, "scenarios", "pyproject_project_def"), package) + pyproject = package / "pyproject.toml" + contents = pyproject.read_text().replace("setuptools.build_meta", "maturin") + contents = contents.replace('requires = ["setuptools>=61.0.0", "wheel"]', 'requires = ["maturin"]') + pyproject.write_text(contents + '\n[tool.cibuildwheel]\nbuild = "cp310-*"\n') + + destination = tmp_path / "dist" + create_package(str(package), str(destination), enable_wheel=False) + + assert [command for command, _ in build_commands] == [[sys.executable, "-m", "build", "-s", "-o", str(destination)]] + assert build_commands[0][1]["check"] is True + + +def test_cibuildwheel_failure_propagates(tmp_path, monkeypatch): + package = tmp_path / "package" + shutil.copytree(os.path.join(integration_folder, "scenarios", "pyproject_project_def_with_extension"), package) + + def fail_build(command, **kwargs): + raise subprocess.CalledProcessError(1, command) + + monkeypatch.setattr("ci_tools.build.run_logged", fail_build) + with pytest.raises(subprocess.CalledProcessError): + create_package(str(package), str(tmp_path / "dist")) diff --git a/eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py b/eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py new file mode 100644 index 000000000000..dcdbe788b7dd --- /dev/null +++ b/eng/tools/azure-sdk-tools/tests/test_native_build_pipeline.py @@ -0,0 +1,222 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import json +import os +from pathlib import Path +import re +import shutil +import subprocess + +import pytest +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[4] +TEMPLATES = REPO_ROOT / "eng" / "pipelines" / "templates" +RUST_GATE = "${{ if parameters.InstallMsRustToolchain }}" +RUST_PARAMETERS = ( + "InstallMsRustToolchain", + "MsRustWorkingDirectory", + "MsRustToolchainFeed", + "MsRustAdditionalTargets", +) + + +def load_template(relative_path): + return yaml.safe_load((TEMPLATES / relative_path).read_text()) + + +def run_powershell(script, **environment): + powershell = shutil.which("pwsh") + if not powershell: + pytest.skip("PowerShell is required to execute pipeline scripts") + return subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", script], + env={**os.environ, **environment}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + +def test_rust_support_is_disabled_by_default_and_forwarded(): + archetype = load_template("stages/archetype-sdk-client.yml") + jobs = load_template("jobs/ci.yml") + build = load_template("steps/build-package-artifacts.yml") + for template in (archetype, jobs, build): + defaults = { + parameter["name"]: parameter["default"] + for parameter in template["parameters"] + if parameter["name"] in RUST_PARAMETERS + } + assert defaults["InstallMsRustToolchain"] is False + assert all(defaults[name] == "" for name in RUST_PARAMETERS[1:]) + + build_stage = next(stage for stage in archetype["extends"]["parameters"]["stages"] if stage.get("stage") == "Build") + forwarded = build_stage["jobs"][0]["parameters"] + assert all(forwarded[name] == "${{ parameters." + name + " }}" for name in RUST_PARAMETERS) + + for job in jobs["jobs"]: + if job.get("job") not in ("Build_Linux", "Build_Windows", "Build_MacOS"): + continue + assert job[RUST_GATE]["timeoutInMinutes"] == 240 + assert job["${{ else }}"]["timeoutInMinutes"] == 90 + forwarded = job["steps"][0]["parameters"] + assert all(forwarded[name] == "${{ parameters." + name + " }}" for name in RUST_PARAMETERS) + + +def test_rust_credentials_and_target_setup_are_opt_in(): + build = load_template("steps/build-package-artifacts.yml") + install = next(step[RUST_GATE][0] for step in build["steps"] if RUST_GATE in step) + assert install["template"].endswith("install-msrust-toolchain.yml") + assert install["parameters"]["Condition"] == "and(succeeded(), eq(variables['ENABLE_EXTENSION_BUILD'], 'true'))" + assert install["parameters"]["ToolchainFeed"] == "${{ parameters.MsRustToolchainFeed }}" + assert install["parameters"]["${{ if eq(parameters.ArtifactSuffix, 'windows') }}"] == { + "AdditionalTargets": "${{ parameters.MsRustAdditionalTargets }}" + } + + generate = next(step for step in build["steps"] if step.get("displayName") == "Generate Packages") + assert generate[RUST_GATE]["timeoutInMinutes"] == 210 + assert generate["${{ else }}"]["timeoutInMinutes"] == 80 + assert generate["env"]["LOGLEVEL"] == "$(SDK_BUILD_LOGLEVEL)" + assert generate["env"][RUST_GATE]["MSRUSTUP_ACCESS_TOKEN"] == "$(System.AccessToken)" + assert "MSRUSTUP_ACCESS_TOKEN" not in generate["env"] + + installer = load_template("steps/install-msrust-toolchain.yml") + assert all(step["condition"] == "${{ parameters.Condition }}" for step in installer["steps"]) + cargo_auth = next(step for step in installer["steps"] if step.get("task") == "CargoAuthenticate@0") + assert cargo_auth["env"] == {"SYSTEM_ACCESSTOKEN": "$(VSS_NUGET_ACCESSTOKEN)"} + + provision = next( + step for step in build["steps"] if step.get("displayName") == "Pre-provision CPython for cibuildwheel" + ) + assert provision["env"]["INSTALL_MS_RUST_TOOLCHAIN"] == "${{ format('{0}', parameters.InstallMsRustToolchain) }}" + + +@pytest.mark.parametrize("sign_binaries", [False, True]) +def test_platform_selection_uses_artifact_opt_in(tmp_path, sign_binaries): + package_info = {"Name": "example-native-package", "ArtifactDetails": {"signBinaries": sign_binaries}} + (tmp_path / "example-native-package.json").write_text(json.dumps(package_info)) + resolver = load_template("steps/resolve-build-platforms.yml")["steps"][0]["pwsh"] + resolver = resolver.replace("${{ parameters.PackagePropertiesFolder }}", str(tmp_path)) + + result = run_powershell(resolver) + + assert result.returncode == 0, result.stderr + assert ("variable=ENABLE_EXTENSION_BUILD]true" in result.stdout) is sign_binaries + assert f"variable=SDK_BUILD_LOGLEVEL]{'DEBUG' if sign_binaries else 'INFO'}" in result.stdout + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_container_forwarding_preserves_non_rust_builds(enabled): + build = load_template("steps/build-package-artifacts.yml") + forwarding = next( + step for step in build["steps"] if step.get("displayName") == "Forward build environment to cibuildwheel" + ) + extra = forwarding["env"][RUST_GATE]["RUST_ENVIRONMENT_PASS"] if enabled else "" + + result = run_powershell(forwarding["pwsh"], RUST_ENVIRONMENT_PASS=extra) + + assert result.returncode == 0, result.stderr + assert "variable=CIBW_ENVIRONMENT_PASS_LINUX]PIP_INDEX_URL" in result.stdout + assert ("MSRUSTUP_ACCESS_TOKEN" in result.stdout) is enabled + assert ("CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_TOKEN" in result.stdout) is enabled + + +@pytest.mark.parametrize("missing", ["feed", "directory", "toolchain"]) +def test_rust_installation_requires_explicit_configuration(tmp_path, missing): + validation = load_template("steps/install-msrust-toolchain.yml")["steps"][0]["pwsh"] + if missing != "toolchain": + (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "ms-prod-1.97"\n') + result = run_powershell( + validation, + RUST_TOOLCHAIN_FEED=( + "" if missing == "feed" else "https://pkgs.dev.azure.com/azure-sdk/_packaging/example/nuget/v3/index.json" + ), + RUST_WORKING_DIRECTORY="" if missing == "directory" else str(tmp_path), + ) + assert result.returncode != 0 + + +def test_rust_installation_accepts_explicit_configuration(tmp_path): + validation = load_template("steps/install-msrust-toolchain.yml")["steps"][0]["pwsh"] + (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "ms-prod-1.97"\n') + result = run_powershell( + validation, + RUST_TOOLCHAIN_FEED="https://pkgs.dev.azure.com/azure-sdk/_packaging/example/nuget/v3/index.json", + RUST_WORKING_DIRECTORY=str(tmp_path), + ) + assert result.returncode == 0, result.stderr + + +def test_cargo_feed_configuration_does_not_overwrite_existing_config(tmp_path): + configure = load_template("steps/install-msrust-toolchain.yml")["steps"][2]["pwsh"] + feed = "sparse+https://pkgs.dev.azure.com/azure-sdk/public/_packaging/example/Cargo/index/" + result = run_powershell( + configure, CARGO_HOME=str(tmp_path), CARGO_FEED_URL=feed, RUST_TOOLCHAIN_FEED="example-feed" + ) + assert result.returncode == 0, result.stderr + config = (tmp_path / "config.toml").read_text() + assert feed in config + assert 'replace-with = "azure-sdk-for-rust-public"' in config + assert "token" not in config + assert "variable=CARGO_REGISTRIES_AZURE_SDK_FOR_RUST_PUBLIC_INDEX]" + feed in result.stdout + + result = run_powershell( + configure, CARGO_HOME=str(tmp_path), CARGO_FEED_URL=feed, RUST_TOOLCHAIN_FEED="example-feed" + ) + assert result.returncode != 0 + assert (tmp_path / "config.toml").read_text() == config + + +def test_cargo_feed_configuration_uses_default_cargo_home(tmp_path): + configure = load_template("steps/install-msrust-toolchain.yml")["steps"][2]["pwsh"] + feed = "sparse+https://pkgs.dev.azure.com/azure-sdk/public/_packaging/example/Cargo/index/" + + result = run_powershell( + configure, + CARGO_HOME="", + CARGO_FEED_URL=feed, + HOME=str(tmp_path), + RUST_TOOLCHAIN_FEED="example-feed", + ) + + assert result.returncode == 0, result.stderr + config = tmp_path / ".cargo" / "config.toml" + assert config.is_file() + assert feed in config.read_text() + + +def test_build_powershell_scripts_parse(): + for template_path in ( + "steps/install-msrust-toolchain.yml", + "steps/build-package-artifacts.yml", + "steps/resolve-build-platforms.yml", + ): + template = load_template(template_path) + for step in template["steps"]: + if "pwsh" not in step: + continue + # Azure expands template expressions and the command-valued PIP_EXE macro before parsing. + script = re.sub(r"\$\{\{.*?\}\}", "example", step["pwsh"]) + script = script.replace("$(PIP_EXE)", "python -m pip") + result = run_powershell( + """ + $tokens = $null + $errors = $null + [System.Management.Automation.Language.Parser]::ParseInput( + $env:SCRIPT, [ref]$tokens, [ref]$errors + ) | Out-Null + if ($errors.Count) { + $errors | ForEach-Object { Write-Error $_.Message } + exit 1 + } + """, + SCRIPT=script, + ) + assert result.returncode == 0, f"{template_path}: {step['displayName']}: {result.stderr}" diff --git a/eng/tools/azure-sdk-tools/tests/test_parse_functionality.py b/eng/tools/azure-sdk-tools/tests/test_parse_functionality.py index 1405850a0285..705261fde8ec 100644 --- a/eng/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/eng/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -1,6 +1,8 @@ from ci_tools.parsing import parse_require, ParsedSetup +from ci_tools.parsing.parse_functions import has_cibuildwheel_config from packaging.specifiers import SpecifierSet import os +import shutil from unittest.mock import patch import pytest @@ -333,3 +335,52 @@ def test_namespace_discovery_with_substantial_content(): assert result == "test.module" finally: os.unlink(temp_file) + + +@pytest.mark.parametrize( + "config", + [ + "\n[tool.cibuildwheel]\n", + '\n[tool.cibuildwheel]\nbuild = "cp310-*"\n', + '\n[tool.cibuildwheel.linux]\narchs = ["x86_64", "aarch64"]\n', + ], +) +def test_cibuildwheel_config_detected_without_ext_modules(tmp_path, config): + package = tmp_path / "package" + shutil.copytree(pyproject_scenario, package) + with (package / "pyproject.toml").open("a") as stream: + stream.write(config) + + parsed = ParsedSetup.from_path(str(package)) + assert parsed.ext_modules == [] + assert parsed.uses_cibuildwheel is True + assert has_cibuildwheel_config(str(package)) is True + + +@pytest.mark.parametrize("scenario", [pyproject_scenario, setup_project_scenario]) +def test_cibuildwheel_config_absent(scenario): + assert has_cibuildwheel_config(scenario) is False + assert ParsedSetup.from_path(scenario).uses_cibuildwheel is False + + +def test_cibuildwheel_config_detected_for_legacy_setup(tmp_path): + package = tmp_path / "package" + shutil.copytree(setup_project_scenario, package) + (package / "pyproject.toml").write_text('[tool.cibuildwheel]\nbuild = "cp310-*"\n') + + parsed = ParsedSetup.from_path(str(package)) + assert parsed.is_pyproject is False + assert parsed.uses_cibuildwheel is True + + +def test_invalid_cibuildwheel_toml_is_not_ignored(tmp_path): + (tmp_path / "pyproject.toml").write_text("[tool.cibuildwheel\n") + with pytest.raises(ValueError): + has_cibuildwheel_config(str(tmp_path)) + + +@pytest.mark.parametrize("value", ["false", '"not-a-table"']) +def test_cibuildwheel_config_requires_a_table(tmp_path, value): + (tmp_path / "pyproject.toml").write_text(f"[tool]\ncibuildwheel = {value}\n") + with pytest.raises(ValueError, match=r"Expected a \[tool.cibuildwheel\] table"): + has_cibuildwheel_config(str(tmp_path))