From 0efa9c37315e4fc4ba3172e3b55c251eb86e27ba Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Jul 2026 10:35:58 -0700 Subject: [PATCH 01/11] Enable scanning of azure-storage-extensions --- eng/pipelines/templates/stages/1es-redirect.yml | 6 +++++- sdk/storage/ci.yml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 00911bc510b8..ee8f86897cf9 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -19,6 +19,10 @@ parameters: - name: oneESTemplateTag type: string default: release +- name: EnableCompiledCodeql + type: boolean + default: false + extends: ${{ if and(parameters.Use1ESOfficial, eq(parameters.oneESTemplateTag, 'canary')) }}: @@ -56,7 +60,7 @@ extends: justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azur 19 e-sdk/internal/_build/results?buildId=3556850" codeql: compiled: - enabled: false + enabled: ${{ parameters.EnableCompiledCodeql }} justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job." componentgovernance: enabled: false diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 390eab91bb75..d35537c2ab07 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -39,6 +39,9 @@ extends: ServiceDirectory: storage TestProxy: true TestTimeoutInMinutes: 120 + # Enable Compiled CodeQL becuase azure-storage-extensions has C code that + # must be scanned + EnableCompiledCodeql: true Artifacts: - name: azure-storage-blob safeName: azurestorageblob From 52d86ad0f83aae3445f312118efdc2f23f198518 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Jul 2026 10:39:04 -0700 Subject: [PATCH 02/11] param passthrough --- eng/pipelines/templates/stages/archetype-sdk-client.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 53efd1ab54b0..109f9e6bd012 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -81,6 +81,9 @@ parameters: - name: oneESTemplateTag type: string default: release + - name: EnableCompiledCodeql + type: boolean + default: false - name: EnvVars type: object default: {} @@ -89,6 +92,7 @@ extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: oneESTemplateTag: ${{ parameters.oneESTemplateTag }} + EnableCompiledCodeql: ${{ parameters.EnableCompiledCodeql }} stages: - ${{ if and(eq(parameters.SkipPrValidation, true), eq(variables['Build.Reason'], 'Manual')) }}: - stage: NoOp From 0876a03c6b4f83802d0915d516bbed4a37ce0a85 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Jul 2026 11:12:12 -0700 Subject: [PATCH 03/11] Port build changes needed to run cibuildwheel --- eng/tools/azure-sdk-tools/ci_tools/build.py | 90 ++++++++++++------- .../ci_tools/parsing/parse_functions.py | 19 ++++ 2 files changed, 77 insertions(+), 32 deletions(-) diff --git a/eng/tools/azure-sdk-tools/ci_tools/build.py b/eng/tools/azure-sdk-tools/ci_tools/build.py index 26ac6fcfd15a..9aea4d49258c 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/build.py +++ b/eng/tools/azure-sdk-tools/ci_tools/build.py @@ -221,8 +221,16 @@ def create_package( setup_directory_or_file: str, dest_folder: str, enable_wheel: bool = True, enable_sdist: bool = True ): """ - Uses the invoking python executable to build a wheel and sdist file given a setup.py or setup.py directory. Outputs - into a distribution directory and defaults to the value of get_artifact_directory(). + Builds a wheel and/or sdist file given a setup.py, pyproject.toml, or directory containing either. + + For packages with compiled extensions (ext_modules): + - setup.py: uses cibuildwheel to build platform-specific wheels + - pyproject.toml: uses cibuildwheel to build platform-specific wheels (respects [tool.cibuildwheel] config) + + For pure Python packages: + - Uses python -m build + + Outputs into a distribution directory and defaults to get_artifact_directory(). """ dist = get_artifact_directory(dest_folder) @@ -231,36 +239,54 @@ def create_package( should_log_build_output = logger.getEffectiveLevel() <= logging.DEBUG if setup_parsed.is_pyproject: - # when building with pyproject, we will use `python -m build` to build the package - # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully - # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail - # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list - - # given the additional requirements of the package, we should install them in the current environment before attempting to build the package - # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0` - pip_output = get_pip_list_output(sys.executable) - necessary_install_requirements = [ - req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys() - ] - run_logged( - [sys.executable, "-m", "pip", "install", *necessary_install_requirements], - cwd=setup_parsed.folder, - check=False, - should_stream_to_console=should_log_build_output, - ) - run_logged( - [ - sys.executable, - "-m", - "build", - f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", - "-o", - dist, - ], - cwd=setup_parsed.folder, - check=True, - should_stream_to_console=should_log_build_output, - ) + # when building with pyproject, check if package has compiled extensions + if enable_wheel and setup_parsed.ext_modules: + # Use cibuildwheel for compiled extensions (respects [tool.cibuildwheel] config) + run_logged( + [sys.executable, "-vv", "-m", "cibuildwheel", "--output-dir", dist], + cwd=setup_parsed.folder, + check=True, + should_stream_to_console=should_log_build_output, + ) + if enable_sdist: + # Build sdist separately with python -m build + run_logged( + [sys.executable, "-m", "build", "-s", "-o", dist], + cwd=setup_parsed.folder, + check=True, + should_stream_to_console=should_log_build_output, + ) + else: + # Use python -m build for pure Python packages + # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully + # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail + # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list + + # given the additional requirements of the package, we should install them in the current environment before attempting to build the package + # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0` + pip_output = get_pip_list_output(sys.executable) + necessary_install_requirements = [ + req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys() + ] + run_logged( + [sys.executable, "-m", "pip", "install", *necessary_install_requirements], + cwd=setup_parsed.folder, + check=False, + should_stream_to_console=should_log_build_output, + ) + run_logged( + [ + sys.executable, + "-m", + "build", + f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", + "-o", + dist, + ], + cwd=setup_parsed.folder, + check=True, + should_stream_to_console=should_log_build_output, + ) else: if enable_wheel: if setup_parsed.ext_modules: diff --git a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index a8e112434323..2830ee15ae70 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -703,6 +703,25 @@ def parse_pyproject( ext_modules = get_value_from_dict(toml_dict, "tool.setuptools.ext-modules", []) ext_modules = [Extension(**moduleArgDict) for moduleArgDict in ext_modules] + # Declaring ext-modules in [tool.setuptools.ext-modules] is still experimental in setuptools, and the + # abi3 / py_limited_api wheel tag cannot be expressed declaratively, so compiled-extension packages keep + # their Extension(...) definition in setup.py. When such a package also has a [project] table, pyproject + # wins over setup.py during parsing and the extension would otherwise go undetected, causing the build to + # be mis-routed to `python -m build` (pure-Python) instead of cibuildwheel. Fall back to setup.py here so + # the extension is still discovered. Guarded so a setup.py that cannot be parsed cannot regress packages + # that parse fine today. + if not ext_modules: + sibling_setup_py = os.path.join(package_directory, "setup.py") + if os.path.exists(sibling_setup_py): + try: + setup_py_result = parse_setup_py(sibling_setup_py) + ext_package = ext_package or setup_py_result[11] # ext_package + ext_modules = setup_py_result[12] # ext_modules + except Exception as e: # pragma: no cover - defensive, preserves prior behavior + logging.warning( + f"Found setup.py alongside {pyproject_filename} but could not parse it for ext_modules: {e}" + ) + # fmt: off return ( name, # str From bf29e32a6bcebfb206f0b57bf2cfc384d06e89f5 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 15 Jul 2026 12:23:59 -0700 Subject: [PATCH 04/11] Specify langauge --- eng/pipelines/templates/stages/1es-redirect.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index ee8f86897cf9..367ca4cec87f 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -59,9 +59,16 @@ extends: enabled: false justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azur 19 e-sdk/internal/_build/results?buildId=3556850" codeql: - compiled: - enabled: ${{ parameters.EnableCompiledCodeql }} - justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job." + ${{ if eq(parameters.EnableCompiledCodeql, true) }}: + # "cpp" covers both C and C++ code. Language is specified because + # checkout happens after the injected "CodeQL Initialize" step + language: cpp + compiled: + enabled: true + ${{ else }}: + compiled: + enabled: false + justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job." componentgovernance: enabled: false justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job." From c638b41f7dbf045c930c3aa509ef3af31e00855b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:28:59 +0000 Subject: [PATCH 05/11] Fix black formatting: remove trailing whitespace in docstring, remove -vv flag Co-authored-by: danieljurek <2158838+danieljurek@users.noreply.github.com> --- eng/tools/azure-sdk-tools/ci_tools/build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/tools/azure-sdk-tools/ci_tools/build.py b/eng/tools/azure-sdk-tools/ci_tools/build.py index 9aea4d49258c..44177c20c026 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/build.py +++ b/eng/tools/azure-sdk-tools/ci_tools/build.py @@ -222,14 +222,14 @@ def create_package( ): """ Builds a wheel and/or sdist file given a setup.py, pyproject.toml, or directory containing either. - + For packages with compiled extensions (ext_modules): - setup.py: uses cibuildwheel to build platform-specific wheels - pyproject.toml: uses cibuildwheel to build platform-specific wheels (respects [tool.cibuildwheel] config) - + For pure Python packages: - Uses python -m build - + Outputs into a distribution directory and defaults to get_artifact_directory(). """ @@ -243,7 +243,7 @@ def create_package( if enable_wheel and setup_parsed.ext_modules: # Use cibuildwheel for compiled extensions (respects [tool.cibuildwheel] config) run_logged( - [sys.executable, "-vv", "-m", "cibuildwheel", "--output-dir", dist], + [sys.executable, "-m", "cibuildwheel", "--output-dir", dist], cwd=setup_parsed.folder, check=True, should_stream_to_console=should_log_build_output, From 35f42fe360fe05a085229c0a8e6fd22f14686726 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 20 Jul 2026 10:30:03 -0700 Subject: [PATCH 06/11] FIx verbosity, fix typo --- sdk/storage/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index d35537c2ab07..a05078529031 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -39,7 +39,7 @@ extends: ServiceDirectory: storage TestProxy: true TestTimeoutInMinutes: 120 - # Enable Compiled CodeQL becuase azure-storage-extensions has C code that + # Enable Compiled CodeQL because azure-storage-extensions has C code that # must be scanned EnableCompiledCodeql: true Artifacts: From a8b3b2b7cd3a51506b428087e0314eb05f8acf9e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 20 Jul 2026 11:07:35 -0700 Subject: [PATCH 07/11] Test CFS fixes for cibuildwheel --- .../steps/build-package-artifacts.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 6a5499104942..5d9d02ea36f6 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -136,6 +136,35 @@ steps: displayName: 'Install QEMU Dependencies' condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + # Windows: cibuildwheel fetches CPython from api.nuget.org, which is blocked. + # Pre-provision the interpreters from the Azure DevOps NuGet feed into the + # cibuildwheel cache so it skips the blocked download. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to NuGet feed' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['ENABLE_EXTENSION_BUILD'], 'true')) + + - pwsh: | + $ErrorActionPreference = 'Stop' + $feed = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json" + $version = "3.10.11" + $cache = Join-Path "$(Agent.TempDirectory)" "cibw-cache" + $nugetCpython = Join-Path $cache "nuget-cpython" + New-Item -ItemType Directory -Force -Path $nugetCpython | Out-Null + $nuget = Join-Path $cache "nuget.exe" + Invoke-WebRequest -Uri "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -OutFile $nuget + & $nuget install python -Version $version -Source $feed -OutputDirectory $nugetCpython + & $nuget install pythonarm64 -Version $version -Source $feed -OutputDirectory $nugetCpython + Write-Host "##vso[task.setvariable variable=CIBW_CACHE_PATH]$cache" + displayName: 'Pre-provision CPython for cibuildwheel' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['ENABLE_EXTENSION_BUILD'], 'true')) + + # Linux: Set CIBW_ENVIRONMENT to use PIP_INDEX_URL to fetch dependencies from + # the Azure DevOps feed instead of PyPI. + - pwsh: | + Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT]PIP_INDEX_URL=$(PIP_INDEX_URL)" + displayName: 'Set CIBW_ENVIRONMENT' + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + - pwsh: | which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive --service="${{ parameters.ServiceDirectory }}" From 3891b46e13fb444b98c1e8e4765dbb2fdc9fec41 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 20 Jul 2026 11:23:46 -0700 Subject: [PATCH 08/11] Use agent's included nuget version --- eng/pipelines/templates/steps/build-package-artifacts.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 5d9d02ea36f6..d50ab9337655 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -150,8 +150,11 @@ steps: $cache = Join-Path "$(Agent.TempDirectory)" "cibw-cache" $nugetCpython = Join-Path $cache "nuget-cpython" New-Item -ItemType Directory -Force -Path $nugetCpython | Out-Null - $nuget = Join-Path $cache "nuget.exe" - Invoke-WebRequest -Uri "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -OutFile $nuget + $nuget = (Get-Command nuget -ErrorAction SilentlyContinue).Source + if (-not $nuget) { + $nuget = Join-Path $cache "nuget.exe" + Invoke-WebRequest -Uri "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -OutFile $nuget + } & $nuget install python -Version $version -Source $feed -OutputDirectory $nugetCpython & $nuget install pythonarm64 -Version $version -Source $feed -OutputDirectory $nugetCpython Write-Host "##vso[task.setvariable variable=CIBW_CACHE_PATH]$cache" From d9c3905dffe6718518d37d51b26e17a752cf8446 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 20 Jul 2026 11:39:36 -0700 Subject: [PATCH 09/11] Assume nuget is present --- eng/pipelines/templates/steps/build-package-artifacts.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index d50ab9337655..b1f80420ff85 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -150,11 +150,7 @@ steps: $cache = Join-Path "$(Agent.TempDirectory)" "cibw-cache" $nugetCpython = Join-Path $cache "nuget-cpython" New-Item -ItemType Directory -Force -Path $nugetCpython | Out-Null - $nuget = (Get-Command nuget -ErrorAction SilentlyContinue).Source - if (-not $nuget) { - $nuget = Join-Path $cache "nuget.exe" - Invoke-WebRequest -Uri "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" -OutFile $nuget - } + $nuget = (Get-Command nuget).Source & $nuget install python -Version $version -Source $feed -OutputDirectory $nugetCpython & $nuget install pythonarm64 -Version $version -Source $feed -OutputDirectory $nugetCpython Write-Host "##vso[task.setvariable variable=CIBW_CACHE_PATH]$cache" From cc84d4a8bcb6894ea3958a80410a270cc5c445ea Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 20 Jul 2026 11:44:31 -0700 Subject: [PATCH 10/11] Error handling --- eng/pipelines/templates/steps/build-package-artifacts.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index b1f80420ff85..d3ebe40949fe 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -152,7 +152,15 @@ steps: New-Item -ItemType Directory -Force -Path $nugetCpython | Out-Null $nuget = (Get-Command nuget).Source & $nuget install python -Version $version -Source $feed -OutputDirectory $nugetCpython + if ($LASTEXITCODE) { + Write-Error "Failed to download python $version" + exit 1 + } & $nuget install pythonarm64 -Version $version -Source $feed -OutputDirectory $nugetCpython + if ($LASTEXITCODE) { + Write-Error "Failed to download pythonarm64 $version" + exit 1 + } Write-Host "##vso[task.setvariable variable=CIBW_CACHE_PATH]$cache" displayName: 'Pre-provision CPython for cibuildwheel' condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['ENABLE_EXTENSION_BUILD'], 'true')) From 34e2abb16f9119ba173b7da00bb31b4338624216 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 20 Jul 2026 11:45:57 -0700 Subject: [PATCH 11/11] Pass in PIP_INDEX_URL --- .../templates/steps/build-package-artifacts.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index d3ebe40949fe..3cc3b0711b67 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -165,11 +165,13 @@ steps: displayName: 'Pre-provision CPython for cibuildwheel' condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['ENABLE_EXTENSION_BUILD'], 'true')) - # Linux: Set CIBW_ENVIRONMENT to use PIP_INDEX_URL to fetch dependencies from - # the Azure DevOps feed instead of PyPI. + # Linux: forward PIP_INDEX_URL into the build container so pip fetches build + # dependencies from the Azure DevOps feed instead of PyPI. Using + # CIBW_ENVIRONMENT_PASS_LINUX (not CIBW_ENVIRONMENT) preserves the package's + # own [tool.cibuildwheel].environment (CFLAGS) setting. - pwsh: | - Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT]PIP_INDEX_URL=$(PIP_INDEX_URL)" - displayName: 'Set CIBW_ENVIRONMENT' + Write-Host "##vso[task.setvariable variable=CIBW_ENVIRONMENT_PASS_LINUX]PIP_INDEX_URL" + displayName: 'Forward PIP_INDEX_URL to cibuildwheel' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - pwsh: |