From b8456215dafd9c5ef1ca0de4af7c37c5c04f88e7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:27:59 +0530 Subject: [PATCH 01/35] Add OneBranch release pipelines for mssql-python-odbc Adds the build and release pipelines for the standalone mssql-python-odbc package: per-platform build stages (Windows x64/arm64, macOS universal2, Linux manylinux_2_28 + musllinux_1_2 for x86_64/aarch64), a consolidate job that gathers all 7 py3-none wheels, the build orchestration pipeline, and the ESRP-based official release pipeline. The build stages run no compilation (data-only package); Linux builds run in tdslibrs ACR PyPA containers and the musllinux tag is auto-detected via libc_ver. Infra follow-ups (ESRP onboarding of the mssql-python-odbc name, registering the ADO build definition to set odbcBuildDefinitionId, and sharing the ESRP variable group) are marked with TODO(infra) comments. --- .../build-release-odbc-pipeline.yml | 210 ++++++++++++++++ .../jobs/consolidate-odbc-artifacts-job.yml | 68 ++++++ .../official-release-odbc-pipeline.yml | 227 ++++++++++++++++++ .../stages/build-odbc-linux-stage.yml | 163 +++++++++++++ .../stages/build-odbc-macos-stage.yml | 107 +++++++++ .../stages/build-odbc-windows-stage.yml | 125 ++++++++++ 6 files changed, 900 insertions(+) create mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml create mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/stages/build-odbc-linux-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-macos-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-windows-stage.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml new file mode 100644 index 000000000..260fd1c3c --- /dev/null +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -0,0 +1,210 @@ +# ========================================================================================= +# OneBranch Build Pipeline for mssql-python-odbc +# ========================================================================================= +# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. +# +# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is +# NO compiled Python extension, so every wheel is tagged `py3-none-` and a +# SINGLE build per platform covers all supported Python versions (3.10+). That is why +# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces +# only 7 wheels total: +# - Windows: x64, arm64 +# - macOS: universal2 +# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) +# +# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml +# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). +# +# INFRA PREREQUISITES (see companion release pipeline for the definition ID): +# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). +# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. +# - Ensure the reused pools/images are available: Python-1ES-pool +# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the +# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. +# ========================================================================================= + +# Build number format: YYDDD.r +name: $(Year:YY)$(DayOfYear)$(Rev:.r) + +# Only trigger when ODBC-package inputs change (path filters are literal prefixes, +# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. +trigger: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +pr: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +parameters: + - name: oneBranchType + displayName: 'OneBranch Template Type' + type: string + values: + - 'Official' + - 'NonOfficial' + default: 'NonOfficial' + + - name: runSdlTasks + displayName: 'Run SDL Security Tasks' + type: boolean + default: true + + # Windows: one wheel per architecture (NO per-Python matrix). + - name: odbcWindowsConfigs + type: object + default: + - arch: 'x64' + - arch: 'arm64' + + # macOS: single universal2 wheel. + - name: odbcMacosConfigs + type: object + default: + - name: 'universal2' + + # Linux: one wheel per libc/arch combo. + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + +variables: + # Scheduled builds run as Official; manual/PR builds use the parameter. + - name: effectiveOneBranchType + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'Official' + ${{ else }}: + value: '${{ parameters.oneBranchType }}' + + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the + # per-stage EsrpMalwareScanning tasks. + - template: /OneBranchPipelines/variables/signing-variables.yml@self + - group: 'ESRP Federated Creds (AME)' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + +extends: + template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + + # This pipeline compiles NO first-party native code — it only packages the + # pre-built ODBC driver binaries, which are signed and governed by the Microsoft + # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; + # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). + binskim: + enabled: false + justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' + apiscan: + enabled: false + justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' + armory: + enabled: false + codeql: + enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' + + credscan: + enabled: ${{ parameters.runSdlTasks }} + policheck: + enabled: ${{ parameters.runSdlTasks }} + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: ${{ parameters.runSdlTasks }} + sbom: + enabled: ${{ parameters.runSdlTasks }} + packageName: 'mssql-python-odbc' + tsa: + enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + # WINDOWS BUILD STAGES (2: x64, arm64) + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # MACOS BUILD STAGE (1: universal2) + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # CONSOLIDATE STAGE (waits for all 7 platform stages) + - stage: Consolidate + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml new file mode 100644 index 000000000..8ac829631 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -0,0 +1,68 @@ +# Consolidate ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-odbc` wheels from all build stages and +# consolidates them into a single dist/ folder for the release pipeline to publish. +# Expected: 7 wheels (2 Windows + 1 macOS universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download ALL artifacts from the current build (one per platform stage). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All ODBC Platform Artifacts' + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" + else + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + fi + displayName: 'Consolidate ODBC wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_Consolidate_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml new file mode 100644 index 000000000..3a9898c25 --- /dev/null +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -0,0 +1,227 @@ +# ========================================================================================= +# OneBranch Official Release Pipeline for mssql-python-odbc +# ========================================================================================= +# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to +# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group +# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python +# release (same inputs and FolderLocation); the only differences are the build pipeline +# source and that this package has no debug symbols to publish. +# +# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the +# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# ========================================================================================= + +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC + +# Manual trigger only — releases should be deliberate. +trigger: none +pr: none + +parameters: + - name: releaseToPyPI + displayName: 'Release to PyPI (Production)' + type: boolean + default: false # Safety: default false to prevent accidental releases. + +variables: + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Shared ESRP identity — same group used by the mssql-python release pipeline. + - group: 'ESRP Federated Creds (AME)' + # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # once it is registered (the mssql-python build definition is 2199). Left as 0 so the + # pipeline compiles; the download tasks will fail at runtime until this is set. + - name: odbcBuildDefinitionId + value: '0' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + + # Reference to the ODBC build pipeline (select a run in the UI when releasing). + pipelines: + - pipeline: buildPipeline + source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + trigger: none + +extends: + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress + suppressionSet: default + + # Artifacts were already scanned during the build; this pipeline only republishes. + binskim: + enabled: false + justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + tsa: + enabled: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + - stage: ReleasePackages + displayName: 'Release mssql-python-odbc to PyPI' + + jobs: + # Job 1: Download + validate the consolidated wheels (custom pool). + - job: PrepAndValidate + displayName: 'Download and Validate Artifacts' + + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Consolidated ODBC Artifacts' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + - task: PowerShell@2 + displayName: 'List and Verify Downloaded Wheels' + inputs: + targetType: 'inline' + script: | + $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" + if (-not (Test-Path $wheelsPath)) { + Write-Error "Wheel directory not found at: $wheelsPath" + exit 1 + } + $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" + Write-Host "Total wheel files found: $($wheels.Count)" + + $allValid = $true + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($wheel in $wheels) { + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) + $entryCount = $zip.Entries.Count + $zip.Dispose() + Write-Host " OK $($wheel.Name) ($entryCount entries)" + } catch { + Write-Error " BAD $($wheel.Name): $_" + $allValid = $false + } + } + if (-not $allValid) { + Write-Error "One or more wheel files are corrupted" + exit 1 + } + if ($wheels.Count -ne 7) { + Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + } + Write-Host "All wheels verified successfully!" + + # Dry-run summary (when release is disabled). + - ${{ if eq(parameters.releaseToPyPI, false) }}: + - task: PowerShell@2 + displayName: 'Dry Run - Release Skipped' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "DRY RUN MODE - No Release Performed" + Write-Host "Package: mssql-python-odbc" + Write-Host "====================================" + Write-Host "To perform an actual release:" + Write-Host "1. Set 'releaseToPyPI' parameter to true" + Write-Host "2. Re-run pipeline" + + # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). + # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. + - ${{ if eq(parameters.releaseToPyPI, true) }}: + - job: PyPIRelease + displayName: 'ESRP Release to PyPI' + dependsOn: PrepAndValidate + + templateContext: + type: releaseJob + isProduction: true + + pool: + type: windows + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Artifacts from ODBC Build Pipeline' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + # ESRP release task — identical to the mssql-python release pipeline + # (same shared ESRP identity and PyPI publish inputs). + - task: EsrpRelease@9 + displayName: 'ESRP Release to PyPI' + inputs: + connectedservicename: '$(ESRPConnectedServiceName)' + usemanagedidentity: true + keyvaultname: '$(AuthAKVName)' + signcertname: '$(AuthSignCertName)' + clientid: '$(EsrpClientId)' + Intent: 'PackageDistribution' + ContentType: 'PyPI' + ContentSource: 'Folder' + FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' + WaitForReleaseCompletion: true + Owners: '$(owner)' + Approvers: '$(approver)' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'ESRPRELPACMAN' + DomainTenantId: '$(DomainTenantId)' + + - task: PowerShell@2 + displayName: 'Release Summary' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "ESRP Release Completed" + Write-Host "Package: mssql-python-odbc" + Write-Host "Target: PyPI" + Write-Host "====================================" + Write-Host "Next steps:" + Write-Host "1. Verify release in the ESRP portal" + Write-Host "2. Wait for approval workflow completion" + Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml new file mode 100644 index 000000000..798b3c341 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -0,0 +1,163 @@ +# ODBC Linux Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE libc/arch combo +# (manylinux_2_28 or musllinux_1_2 x x86_64 or aarch64). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# A PyPA build container is still used so `platform.libc_ver()` reports the right +# libc and the wheel is tagged manylinux_2_28_* or musllinux_1_2_* correctly. +# aarch64 wheels are produced under QEMU emulation. Driver binaries come from the +# committed mssql_python/libs/ tree that is bind-mounted into the container. +parameters: + # Stage identifier (e.g., 'ODBC_Linux_manylinux_2_28_x86_64'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + # Distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc). + - name: linuxTag + type: string + # CPU architecture: 'x86_64' (AMD64) or 'aarch64' (ARM64). + - name: arch + type: string + # Docker platform for QEMU emulation: 'linux/amd64' or 'linux/arm64'. + - name: dockerPlatform + type: string + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Reuse the same custom 1ES Linux pool as the mssql-python build. + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 60 + + variables: + # BinSkim needs ICU libs not present in manylinux/musllinux containers. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + - bash: | + set -e + if ! docker info > /dev/null 2>&1; then + echo "Starting Docker daemon..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Setup and start Docker daemon' + + - script: | + sudo apt-get install -y qemu-user-static + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(ob_outputDirectory)/wheels + mkdir -p $(ob_outputDirectory)/wheels + displayName: 'Prepare artifact directories' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + # Same PyPA images as the mssql-python Linux build (mirrored into ACR). + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'. Expected: manylinux_2_28 or musllinux" >&2 + exit 1 + fi + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $IMAGE tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + # Build the single py3-none wheel inside the container. Any CPython works; no + # compilation runs. setup_odbc.py's get_platform_info() auto-detects musl via + # platform.libc_ver(): on musllinux images it ALWAYS emits musllinux_1_2_ + # and ignores MANYLINUX_TAG. MANYLINUX_TAG is only consulted on glibc images, so + # hardcoding "manylinux_2_28" below is correct for both image families. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + docker exec \ + -e targetArch="$(ARCH)" \ + -e MANYLINUX_TAG="manylinux_2_28" \ + build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -eux + # Pick any available CPython from the PyPA image. + PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) + echo "Using: $($PY --version)" + $PY -m pip install -q -U pip setuptools wheel + cd /workspace + $PY setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh /workspace/dist/*.whl + ' + displayName: 'Build ODBC wheel in container' + + - script: | + set -euxo pipefail + cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Linux Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml new file mode 100644 index 000000000..9b47388e3 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -0,0 +1,107 @@ +# ODBC macOS Single Configuration Stage Template +# Builds the universal2 `mssql-python-odbc` wheel (arm64 + x86_64 driver binaries in +# one wheel). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# `setup_odbc.py` returns the macosx_15_0_universal2 platform tag and copies both +# macOS arch subtrees from mssql_python/libs, so a single build serves both slices. +parameters: + # Stage identifier (e.g., 'ODBC_MacOS_universal2'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + - name: oneBranchType + type: string + default: 'Official' + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC macOS Universal2' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - macOS Universal2' + # macOS pools declare as 'linux' type (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-14' + timeoutInMinutes: 60 + + variables: + # BinSkim targets PE binaries; macOS uses Mach-O, so disable it here. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS stages). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - script: | + set -e + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # get_platform_info() forces the macosx_15_0_universal2 tag; sync_libs() copies + # both arm64 + x86_64 driver subtrees, so no post-build retag is required. + - script: | + set -e + python setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh dist/*.whl + displayName: 'Build ODBC wheel (universal2)' + + - script: | + set -e + python -m twine check dist/*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)/wheels' + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC macOS Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml new file mode 100644 index 000000000..7a812de45 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -0,0 +1,125 @@ +# ODBC Windows Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE Windows architecture. +# +# Unlike the mssql-python build, this package ships ONLY pre-built ODBC driver +# binaries (data) — there is NO compiled Python extension. A single +# `py3-none-` wheel therefore serves every supported Python version +# (3.10+), so this stage does NOT use a per-Python-version matrix and does NOT +# run pytest against SQL Server. The driver binaries are already committed under +# `mssql_python/libs/`, so `checkout: self` is all that is needed; `setup_odbc.py` +# packages the current platform's subtree. +parameters: + # Stage identifier (e.g., 'ODBC_Win_x64'). + - name: stageName + type: string + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # Target architecture: 'x64' (AMD64) or 'arm64' (ARM64). + - name: architecture + type: string + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheel. + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Windows ${{ parameters.architecture }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - Windows ${{ parameters.architecture }}' + # Reuse the same custom 1ES Windows pool as the mssql-python build. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + # OneBranch output directory for artifacts. + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows stages). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). + targetArch: ${{ parameters.architecture }} + + steps: + # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + # The ARM64 wheel is pure data, so it can be produced on an x64 host — there is + # no native compilation and hence no ARM64 python.lib cross-compile machinery. + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build the platform-specific, Python-agnostic wheel. ARCHITECTURE controls the + # platform tag (win_amd64 / win_arm64). No native compilation happens here — + # setup_odbc.py only packages the driver binaries from mssql_python/libs. + - powershell: | + $ErrorActionPreference = "Stop" + $env:ARCHITECTURE = "${{ parameters.architecture }}" + python setup_odbc.py bdist_wheel + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + displayName: 'Build ODBC wheel (ARCHITECTURE=${{ parameters.architecture }})' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m twine check dist\*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifact' + + # OneBranch requires artifact naming: drop__. + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Windows Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheel for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 From c19a261ed16c625e4f35f9111941600fa9bcd3d4 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:43:11 +0530 Subject: [PATCH 02/35] Address PR review: fail-fast on wheel count, armory justification, Linux twine check + container cleanup, reword infra TODOs --- OneBranchPipelines/build-release-odbc-pipeline.yml | 1 + .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++--- OneBranchPipelines/official-release-odbc-pipeline.yml | 9 +++++---- OneBranchPipelines/stages/build-odbc-linux-stage.yml | 8 +++++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml index 260fd1c3c..b64dc936b 100644 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -144,6 +144,7 @@ extends: justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' armory: enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' codeql: enabled: false justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 8ac829631..0225d34a2 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -54,10 +54,10 @@ jobs: WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then - echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" - else - echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." displayName: 'Consolidate ODBC wheels' - task: PublishPipelineArtifact@1 diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 3a9898c25..90eaedddb 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -8,7 +8,7 @@ # source and that this package has no debug symbols to publish. # # This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# `mssql-python-odbc` project name lands and the two infra values below are set. # ========================================================================================= name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC @@ -28,7 +28,7 @@ variables: - template: /OneBranchPipelines/variables/onebranch-variables.yml@self # Shared ESRP identity — same group used by the mssql-python release pipeline. - group: 'ESRP Federated Creds (AME)' - # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' # once it is registered (the mssql-python build definition is 2199). Left as 0 so the # pipeline compiles; the download tasks will fail at runtime until this is set. - name: odbcBuildDefinitionId @@ -44,7 +44,7 @@ resources: # Reference to the ODBC build pipeline (select a run in the UI when releasing). pipelines: - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. trigger: none extends: @@ -141,7 +141,8 @@ extends: exit 1 } if ($wheels.Count -ne 7) { - Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" + exit 1 } Write-Host "All wheels verified successfully!" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index 798b3c341..d0a9883cd 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -123,9 +123,10 @@ stages: # Pick any available CPython from the PyPA image. PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) echo "Using: $($PY --version)" - $PY -m pip install -q -U pip setuptools wheel + $PY -m pip install -q -U pip setuptools wheel twine cd /workspace $PY setup_odbc.py bdist_wheel + $PY -m twine check /workspace/dist/*.whl echo "Produced wheels:"; ls -lh /workspace/dist/*.whl ' displayName: 'Build ODBC wheel in container' @@ -135,6 +136,11 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + - script: | + docker rm -f build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Stop and remove build container' + condition: always() + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Linux Artifact' inputs: From 62910b4b92233601caa258ead365d7d5417bee92 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:58:20 +0530 Subject: [PATCH 03/35] Address review: preflight guard for odbcBuildDefinitionId placeholder; drop unused targetArch var on Windows stage --- OneBranchPipelines/official-release-odbc-pipeline.yml | 11 +++++++++++ .../stages/build-odbc-windows-stage.yml | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 90eaedddb..ef06b0895 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -99,6 +99,17 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' steps: + - task: PowerShell@2 + displayName: 'Preflight: verify odbcBuildDefinitionId is configured' + inputs: + targetType: 'inline' + script: | + if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { + Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." + exit 1 + } + Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" + - task: DownloadPipelineArtifact@2 displayName: 'Download Consolidated ODBC Artifacts' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 7a812de45..57de838ed 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -48,8 +48,6 @@ stages: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' # OneBranch-required variable (unused on Windows stages). LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' - # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). - targetArch: ${{ parameters.architecture }} steps: # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. From 38b9ff9fbae948078f9e62e3355adee23a98cf17 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:33:46 +0530 Subject: [PATCH 04/35] Fold mssql-python-odbc build+release into existing pipelines Per review feedback, build and release the standalone mssql-python-odbc package from the existing Build-Release-Package-Pipeline (def 2199) and official release pipeline instead of separate ADO definitions. Build pipeline: add odbcWindows/Macos/LinuxConfigs params (2+1+4=7 data-only wheels), reference the 3 odbc stage templates alongside the mssql-python stages, and add a separate ConsolidateOdbc stage publishing drop_ConsolidateOdbc_ConsolidateArtifacts (distinct from mssql-python's drop_Consolidate_ConsolidateArtifacts). Release pipeline: add releasePackage parameter (mssql-python | mssql-python-odbc); select the consolidated artifact accordingly and skip symbol publishing + mssql-py-core version validation for odbc. Delete the redundant standalone orchestrators (build-release-odbc-pipeline.yml, official-release-odbc-pipeline.yml). This removes the need to register a new ADO build definition and the odbcBuildDefinitionId placeholder. Both packages now build from def 2199; the odbc stages consume setup_odbc.py from PR #663, so #663 must merge before these stages can build. --- .../build-release-odbc-pipeline.yml | 211 ---------------- .../build-release-package-pipeline.yml | 94 ++++++- .../jobs/consolidate-odbc-artifacts-job.yml | 5 +- .../official-release-odbc-pipeline.yml | 239 ------------------ .../official-release-pipeline.yml | 56 ++-- 5 files changed, 136 insertions(+), 469 deletions(-) delete mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml delete mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml deleted file mode 100644 index b64dc936b..000000000 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ /dev/null @@ -1,211 +0,0 @@ -# ========================================================================================= -# OneBranch Build Pipeline for mssql-python-odbc -# ========================================================================================= -# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. -# -# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is -# NO compiled Python extension, so every wheel is tagged `py3-none-` and a -# SINGLE build per platform covers all supported Python versions (3.10+). That is why -# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces -# only 7 wheels total: -# - Windows: x64, arm64 -# - macOS: universal2 -# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) -# -# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml -# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). -# -# INFRA PREREQUISITES (see companion release pipeline for the definition ID): -# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). -# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. -# - Ensure the reused pools/images are available: Python-1ES-pool -# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the -# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. -# ========================================================================================= - -# Build number format: YYDDD.r -name: $(Year:YY)$(DayOfYear)$(Rev:.r) - -# Only trigger when ODBC-package inputs change (path filters are literal prefixes, -# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. -trigger: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -pr: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -parameters: - - name: oneBranchType - displayName: 'OneBranch Template Type' - type: string - values: - - 'Official' - - 'NonOfficial' - default: 'NonOfficial' - - - name: runSdlTasks - displayName: 'Run SDL Security Tasks' - type: boolean - default: true - - # Windows: one wheel per architecture (NO per-Python matrix). - - name: odbcWindowsConfigs - type: object - default: - - arch: 'x64' - - arch: 'arm64' - - # macOS: single universal2 wheel. - - name: odbcMacosConfigs - type: object - default: - - name: 'universal2' - - # Linux: one wheel per libc/arch combo. - - name: odbcLinuxConfigs - type: object - default: - - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } - - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } - -variables: - # Scheduled builds run as Official; manual/PR builds use the parameter. - - name: effectiveOneBranchType - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - value: 'Official' - ${{ else }}: - value: '${{ parameters.oneBranchType }}' - - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the - # per-stage EsrpMalwareScanning tasks. - - template: /OneBranchPipelines/variables/signing-variables.yml@self - - group: 'ESRP Federated Creds (AME)' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - -extends: - template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress - suppressionSet: default - - # This pipeline compiles NO first-party native code — it only packages the - # pre-built ODBC driver binaries, which are signed and governed by the Microsoft - # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; - # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). - binskim: - enabled: false - justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' - apiscan: - enabled: false - justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' - armory: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' - codeql: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' - - credscan: - enabled: ${{ parameters.runSdlTasks }} - policheck: - enabled: ${{ parameters.runSdlTasks }} - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: ${{ parameters.runSdlTasks }} - sbom: - enabled: ${{ parameters.runSdlTasks }} - packageName: 'mssql-python-odbc' - tsa: - enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - # WINDOWS BUILD STAGES (2: x64, arm64) - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # MACOS BUILD STAGE (1: universal2) - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # CONSOLIDATE STAGE (waits for all 7 platform stages) - - stage: Consolidate - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 3bd789e85..984e52bcc 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -138,6 +138,33 @@ parameters: - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= + # ODBC PACKAGE CONFIGURATIONS (mssql-python-odbc) + # ========================= + # The standalone mssql-python-odbc package ships ONLY pre-built ODBC driver + # binaries (data) — there is NO compiled extension — so ONE py3-none- + # wheel per platform serves every supported Python version. Hence there is no + # per-Python matrix here: 7 wheels total (2 Windows + 1 macOS + 4 Linux). + # These stages build alongside the mssql-python wheels and are consolidated + # SEPARATELY (see the ConsolidateOdbc stage) so the release pipeline can publish + # mssql-python and mssql-python-odbc independently. + - name: odbcWindowsConfigs + type: object + default: + - { arch: 'x64' } + - { arch: 'arm64' } + - name: odbcMacosConfigs + type: object + default: + - { name: 'universal2' } + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= # PIPELINE VARIABLES # ========================= @@ -338,8 +365,10 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # Total stages: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages - # Stages run in parallel (no dependencies between platform builds) + # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python-odbc: 2 Windows + 1 macOS + 4 Linux + 1 ConsolidateOdbc = 8 stages + # Total: 27 stages. All build stages run in parallel; each Consolidate* stage + # waits only on its own package's build stages. stages: # ========================= # WINDOWS BUILD STAGES @@ -466,3 +495,64 @@ extends: # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging + + # ========================================================================= + # ODBC PACKAGE BUILD STAGES (mssql-python-odbc) + # ========================================================================= + # 7 data-only wheels (no native compile, no pytest) built from setup_odbc.py. + # They run in parallel with the mssql-python stages and are consolidated by a + # SEPARATE ConsolidateOdbc stage so the release pipeline can ship each package + # independently. SDL "just works": these wheels contain the SAME ODBC driver + # binaries already scanned in mssql_python/libs by the main build — no new + # first-party native code is introduced. + # + # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC macOS stage (1): ODBC_MacOS_universal2 + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from + # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release + # pipeline selects one artifact based on its `releasePackage` parameter. + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 0225d34a2..42db132ff 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -64,5 +64,8 @@ jobs: displayName: 'Publish Consolidated ODBC Artifacts' inputs: targetPath: '$(ob_outputDirectory)' - artifact: 'drop_Consolidate_ConsolidateArtifacts' + # Distinct name so it does not collide with the mssql-python consolidate + # artifact (drop_Consolidate_ConsolidateArtifacts) in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateOdbc`. + artifact: 'drop_ConsolidateOdbc_ConsolidateArtifacts' publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml deleted file mode 100644 index ef06b0895..000000000 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ /dev/null @@ -1,239 +0,0 @@ -# ========================================================================================= -# OneBranch Official Release Pipeline for mssql-python-odbc -# ========================================================================================= -# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to -# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group -# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python -# release (same inputs and FolderLocation); the only differences are the build pipeline -# source and that this package has no debug symbols to publish. -# -# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two infra values below are set. -# ========================================================================================= - -name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC - -# Manual trigger only — releases should be deliberate. -trigger: none -pr: none - -parameters: - - name: releaseToPyPI - displayName: 'Release to PyPI (Production)' - type: boolean - default: false # Safety: default false to prevent accidental releases. - -variables: - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Shared ESRP identity — same group used by the mssql-python release pipeline. - - group: 'ESRP Federated Creds (AME)' - # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' - # once it is registered (the mssql-python build definition is 2199). Left as 0 so the - # pipeline compiles; the download tasks will fail at runtime until this is set. - - name: odbcBuildDefinitionId - value: '0' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - - # Reference to the ODBC build pipeline (select a run in the UI when releasing). - pipelines: - - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. - trigger: none - -extends: - template: 'v2/OneBranch.Official.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress - suppressionSet: default - - # Artifacts were already scanned during the build; this pipeline only republishes. - binskim: - enabled: false - justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' - credscan: - enabled: true - policheck: - enabled: true - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: true - tsa: - enabled: true - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - - stage: ReleasePackages - displayName: 'Release mssql-python-odbc to PyPI' - - jobs: - # Job 1: Download + validate the consolidated wheels (custom pool). - - job: PrepAndValidate - displayName: 'Download and Validate Artifacts' - - pool: - type: windows - isCustom: true - name: Python-1ES-pool - demands: - - imageOverride -equals PYTHON-1ES-MMS2022 - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - - steps: - - task: PowerShell@2 - displayName: 'Preflight: verify odbcBuildDefinitionId is configured' - inputs: - targetType: 'inline' - script: | - if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { - Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." - exit 1 - } - Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Consolidated ODBC Artifacts' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - - task: PowerShell@2 - displayName: 'List and Verify Downloaded Wheels' - inputs: - targetType: 'inline' - script: | - $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" - if (-not (Test-Path $wheelsPath)) { - Write-Error "Wheel directory not found at: $wheelsPath" - exit 1 - } - $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" - Write-Host "Total wheel files found: $($wheels.Count)" - - $allValid = $true - Add-Type -AssemblyName System.IO.Compression.FileSystem - foreach ($wheel in $wheels) { - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) - $entryCount = $zip.Entries.Count - $zip.Dispose() - Write-Host " OK $($wheel.Name) ($entryCount entries)" - } catch { - Write-Error " BAD $($wheel.Name): $_" - $allValid = $false - } - } - if (-not $allValid) { - Write-Error "One or more wheel files are corrupted" - exit 1 - } - if ($wheels.Count -ne 7) { - Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" - exit 1 - } - Write-Host "All wheels verified successfully!" - - # Dry-run summary (when release is disabled). - - ${{ if eq(parameters.releaseToPyPI, false) }}: - - task: PowerShell@2 - displayName: 'Dry Run - Release Skipped' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "DRY RUN MODE - No Release Performed" - Write-Host "Package: mssql-python-odbc" - Write-Host "====================================" - Write-Host "To perform an actual release:" - Write-Host "1. Set 'releaseToPyPI' parameter to true" - Write-Host "2. Re-run pipeline" - - # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). - # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. - - ${{ if eq(parameters.releaseToPyPI, true) }}: - - job: PyPIRelease - displayName: 'ESRP Release to PyPI' - dependsOn: PrepAndValidate - - templateContext: - type: releaseJob - isProduction: true - - pool: - type: windows - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download Artifacts from ODBC Build Pipeline' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - # ESRP release task — identical to the mssql-python release pipeline - # (same shared ESRP identity and PyPI publish inputs). - - task: EsrpRelease@9 - displayName: 'ESRP Release to PyPI' - inputs: - connectedservicename: '$(ESRPConnectedServiceName)' - usemanagedidentity: true - keyvaultname: '$(AuthAKVName)' - signcertname: '$(AuthSignCertName)' - clientid: '$(EsrpClientId)' - Intent: 'PackageDistribution' - ContentType: 'PyPI' - ContentSource: 'Folder' - FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' - WaitForReleaseCompletion: true - Owners: '$(owner)' - Approvers: '$(approver)' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMAN' - DomainTenantId: '$(DomainTenantId)' - - - task: PowerShell@2 - displayName: 'Release Summary' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "ESRP Release Completed" - Write-Host "Package: mssql-python-odbc" - Write-Host "Target: PyPI" - Write-Host "====================================" - Write-Host "Next steps:" - Write-Host "1. Verify release in the ESRP portal" - Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index a5ec227f6..731df538e 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -1,5 +1,7 @@ -# OneBranch Official Release Pipeline for mssql-python -# Downloads wheel and symbol artifacts from build pipeline, publishes symbols, and releases wheels to PyPI via ESRP +# OneBranch Official Release Pipeline (mssql-python and mssql-python-odbc) +# Selects the package via the `releasePackage` parameter, downloads that package's +# consolidated wheels (and symbols for mssql-python) from the build pipeline +# (definition 2199), publishes symbols, and releases wheels to PyPI via ESRP. # This pipeline is ALWAYS Official - no NonOfficial option name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release @@ -10,6 +12,16 @@ pr: none # Parameters for release pipeline parameters: + # Which package to release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: 'Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: 'Publish Symbols to Symbol Servers' type: boolean @@ -30,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download/publish based on the target + # package. Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -117,7 +139,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -173,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -215,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -231,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "DRY RUN MODE - No Release Performed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "" Write-Host "Actions performed:" Write-Host "- Downloaded wheels from build pipeline" @@ -273,7 +297,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' - task: EsrpRelease@9 @@ -303,7 +327,7 @@ extends: Write-Host "====================================" Write-Host "ESRP Release Completed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" @@ -313,6 +337,6 @@ extends: Write-Host "Next steps:" Write-Host "1. Verify release in ESRP portal" Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify package on PyPI: https://pypi.org/project/mssql-python/" + Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" From d85fd92bb3d7ec75e47325f30491d629f7273272 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:50:04 +0530 Subject: [PATCH 05/35] Scope consolidate downloads by package to prevent cross-contamination Def 2199 now builds mssql-python and mssql-python-odbc in one run. Both consolidate jobs used DownloadPipelineArtifact@2 with buildType: current, which downloads every artifact in the run and copies all *.whl into dist/. In the merged run this caused: (1) the main Consolidate to sweep the 7 odbc wheels into the mssql-python release (count is warning-only, so silent), and (2) ConsolidateOdbc to see 34 wheels while expecting 7 and hard-fail. Add itemPattern to scope each download to its own package's build-stage artifacts (main: drop_Win_*/drop_MacOS_*/drop_Linux_*; odbc: drop_ODBC_*), giving clean partitioning with no wheel leakage. --- .../jobs/consolidate-artifacts-job.yml | 13 ++++++++++--- .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 478803aa3..336c736a5 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -29,13 +29,20 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from current build - # Matrix jobs publish as: Windows_, macOS_, Linux_ - # This downloads all of them automatically (27 total artifacts) + # Download only the mssql-python platform-stage artifacts from the current build. + # The build definition now ALSO produces mssql-python-odbc artifacts (drop_ODBC_*) + # in the SAME run, so we scope this download with itemPattern to the mssql-python + # stages (drop_Win_*, drop_MacOS_*, drop_Linux_*). Without this, the odbc wheels + # would be swept into this package's dist/ and shipped to the mssql-python PyPI + # project. The odbc wheels are consolidated separately (ConsolidateOdbc stage). - task: DownloadPipelineArtifact@2 displayName: 'Download All Platform Artifacts' inputs: buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** targetPath: '$(Pipeline.Workspace)/all-artifacts' # Consolidate all wheels into single dist/ directory diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 42db132ff..f0d8a0715 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -32,11 +32,15 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from the current build (one per platform stage). + # Download only the mssql-python-odbc platform-stage artifacts (drop_ODBC_*) from + # the current build. The same run also builds the mssql-python wheels (drop_Win_*, + # drop_MacOS_*, drop_Linux_*); itemPattern scopes this download to the odbc stages + # so ONLY the 7 odbc wheels are consolidated here (not the mssql-python wheels). - task: DownloadPipelineArtifact@2 displayName: 'Download All ODBC Platform Artifacts' inputs: buildType: 'current' + itemPattern: 'drop_ODBC_*/**' targetPath: '$(Pipeline.Workspace)/all-artifacts' - bash: | From e3cf2a9a56827bbe9891b2c67d8ff2b0c819b6b1 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:16:59 +0530 Subject: [PATCH 06/35] Correct stale mssql-python wheel count in consolidate diagnostic (27 -> 34) The matrix is 9 Windows + 5 macOS + 4 Linux stages x 5 Python versions (cp310-cp314) = 34 wheels across 18 stages. The diagnostic said 27 (7+4+16), stale from an older 3.10-3.13 / 7-Windows matrix. Count check remains warning-only (no behavior change). --- .../jobs/consolidate-artifacts-job.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 336c736a5..52eb43381 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (15 stages total: 7 Windows + 4 macOS + 4 Linux) -# Each Linux stage builds 4 Python versions, resulting in 27 total wheels +# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) +# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels parameters: - name: oneBranchType type: string @@ -75,12 +75,12 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 27 wheels (7 Windows + 4 macOS + 16 Linux)" + echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" - if [ "$WHEEL_COUNT" -ne 27 ]; then - echo "WARNING: Expected 27 wheels but found $WHEEL_COUNT" + if [ "$WHEEL_COUNT" -ne 34 ]; then + echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" else - echo "SUCCESS: All 27 wheels consolidated!" + echo "SUCCESS: All 34 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' From a4b4e59fc4f9ed09ffa5e79481875339a52159e7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:49:09 +0530 Subject: [PATCH 07/35] Harden odbc release path: pin guard + wheel-content assertions official-release-pipeline.yml: gate 'Published symbols' dry-run/summary text to mssql-python so odbc runs no longer claim symbol publishing that never happens; add a mssql-python-only step asserting the wheel metadata pins mssql-python-odbc==18.6.0 (fails by design if released from a pre-pin build). build-odbc-{linux,macos,windows}-stage.yml: assert each built wheel actually contains the ODBC driver binary (msodbcsql*), guarding against the .gitignore '*.so' trap / silent sync_libs() skip shipping an empty data wheel that still passes twine check. --- .../official-release-pipeline.yml | 48 ++++++++++++++++++- .../stages/build-odbc-linux-stage.yml | 22 +++++++++ .../stages/build-odbc-macos-stage.yml | 21 ++++++++ .../stages/build-odbc-windows-stage.yml | 22 +++++++++ 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 731df538e..7cb529796 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: 'Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "- Downloaded wheels from build pipeline" Write-Host "- Verified wheel integrity" Write-Host "- Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "- Published symbols to SqlClientDrivers org" } Write-Host "" @@ -331,7 +374,8 @@ extends: Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + $symbolsPublished = ("${{ parameters.publishSymbols }}" -eq "True") -and ("${{ parameters.releasePackage }}" -eq "mssql-python") + Write-Host "Symbols Published: $symbolsPublished" Write-Host "=====================================" Write-Host "" Write-Host "Next steps:" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index d0a9883cd..969c0b8b3 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -136,6 +136,28 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + # Guard against the .gitignore '*.so' trap or a silent sync_libs() skip + # shipping a data wheel with NO driver binary (twine check would still pass). + # The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python3 -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - script: | docker rm -f build-$(LINUX_TAG)-$(ARCH) || true displayName: 'Stop and remove build container' diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml index 9b47388e3..c1e7541b1 100644 --- a/OneBranchPipelines/stages/build-odbc-macos-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -79,6 +79,27 @@ stages: TargetFolder: '$(ob_outputDirectory)/wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC macOS Artifact' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 57de838ed..dea809ed0 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -92,6 +92,28 @@ stages: TargetFolder: '$(ob_outputDirectory)\wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver DLL name contains 'msodbcsql' on every platform. + - powershell: | + $ErrorActionPreference = "Stop" + $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No wheels found to verify in $(ob_outputDirectory)\wheels" + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($whl in $wheels) { + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $hasDriver = $zip.Entries | Where-Object { $_.FullName -match 'msodbcsql' } + $zip.Dispose() + if (-not $hasDriver) { + Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary" + } + displayName: 'Assert wheel contains ODBC driver binary' + # OneBranch requires artifact naming: drop__. - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Windows Artifact' From b88e169f006f16bb26f59efc13be907bd11fc93f Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 14:24:25 +0530 Subject: [PATCH 08/35] Hard-fail mssql-python consolidate on wheel-count mismatch Flip the 34-wheel count check from WARNING-only to a hard exit 1, symmetric with the odbc consolidate job's count guard. A mismatch means a build stage silently produced a partial wheel set (e.g. a missing Python version) or the matrix changed without updating the expected count -- previously this shipped silently. --- OneBranchPipelines/jobs/consolidate-artifacts-job.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 52eb43381..40cda12b0 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -77,8 +77,13 @@ jobs: echo "Total wheel count: $WHEEL_COUNT" echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" + # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). + # A mismatch means a build stage silently produced a partial set (e.g. a missing + # Python version) or the build matrix changed without updating this count. + # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. if [ "$WHEEL_COUNT" -ne 34 ]; then - echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" + echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + exit 1 else echo "SUCCESS: All 34 wheels consolidated!" fi From d1b3d467028a4986a3558ce4653a002322484fff Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 14:04:13 +0530 Subject: [PATCH 09/35] odbc: add mssql-python-odbc selector to dummy release pipeline Mirrors the official-release-pipeline odbc changes: releasePackage parameter, consolidatedArtifactName selection for both artifact downloads, and gating the mssql-py-core version validation + symbol publishing to mssql-python only. Stays on Maven ContentType so it never publishes to PyPI. --- OneBranchPipelines/dummy-release-pipeline.yml | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index f9e548fb0..28751e845 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,6 +12,16 @@ pr: none # Parameters for DUMMY release pipeline parameters: + # Which package to test-release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: '[TEST] Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: '[TEST] Publish Symbols to Symbol Servers' type: boolean @@ -32,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download based on the target package. + # Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -114,7 +134,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -175,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: '[TEST] Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # the mssql-python-odbc package ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -217,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (if enabled and symbols exist; mssql-python only) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -233,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DRY RUN MODE ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "" Write-Host "Actions performed:" Write-Host "✓ Downloaded wheels from build pipeline" @@ -280,7 +302,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' # ⚠️ IMPORTANT: Uses Maven ContentType for testing - NOT PyPI! @@ -313,7 +335,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DUMMY RELEASE COMPLETED ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" From 9f93c36073dabd47297265c91527f26efcee4bcf Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:00:10 +0530 Subject: [PATCH 10/35] odbc: align dummy release pipeline mssql-python handling with official Adds the mssql-python-odbc pin guard (gated to mssql-python, matching official-release-pipeline) and gates the 'symbols published' summary messaging on releasePackage==mssql-python so the dummy pipeline exercises the exact same mssql-python validation/messaging as the official pipeline for both packages. --- OneBranchPipelines/dummy-release-pipeline.yml | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 28751e845..c98c351ff 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -196,7 +196,7 @@ extends: Write-Host "=====================================" # Step 3: Validate mssql-py-core is a stable version (mssql-python only; - # the mssql-python-odbc package ships no mssql-py-core.version file) + # mssql-python-odbc ships no mssql-py-core.version file) - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: - task: PowerShell@2 displayName: '[TEST] Validate mssql-py-core is a stable version' @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "✓ Downloaded wheels from build pipeline" Write-Host "✓ Verified wheel integrity" Write-Host "✓ Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Published symbols to SqlClientDrivers org" } Write-Host "✗ ESRP dummy release NOT performed (parameter disabled)" @@ -339,7 +382,7 @@ extends: Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + Write-Host "Symbols Published: $(if ('${{ parameters.publishSymbols }}' -eq 'True' -and '${{ parameters.releasePackage }}' -eq 'mssql-python') { 'True' } else { 'False' })" Write-Host "=====================================" Write-Host "" Write-Host "⚠️ IMPORTANT: This was a DUMMY release using Maven ContentType" @@ -348,7 +391,7 @@ extends: Write-Host "What was tested:" Write-Host "✓ Artifact download from build pipeline" Write-Host "✓ Wheel integrity verification" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Symbol publishing to SqlClientDrivers org" } Write-Host "✓ ESRP release workflow (Maven ContentType)" From 2d93fe07234bf24f30353fd7e56cd6f754a437f9 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:28:24 +0530 Subject: [PATCH 11/35] odbc: bump mssql-python-odbc pin to 18.6.2 in release pipelines --- OneBranchPipelines/dummy-release-pipeline.yml | 2 +- OneBranchPipelines/official-release-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index c98c351ff..6cbb7c44b 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 7cb529796..8937fd0f6 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." From a4e39687fc1930f1c35f17367bb1fe6391e1fe93 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:17:36 +0530 Subject: [PATCH 12/35] build: add buildPackage selector to build pipeline Gate mssql-python and mssql-python-odbc stage groups behind a buildPackage parameter (mssql-python default) so only the selected package builds, halving run time and enabling concurrent per-package builds. Mirrors the releasePackage selector pattern in the release pipelines. --- .../build-release-package-pipeline.yml | 216 ++++++++++-------- 1 file changed, 121 insertions(+), 95 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 984e52bcc..29e735080 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -69,6 +69,22 @@ parameters: displayName: 'Run SDL Security Tasks' type: boolean default: true + + # Which package to build. Both are produced from this same pipeline but are + # fully independent stage groups, so building only one halves the run time and + # lets you run two builds (one per package) concurrently. + # mssql-python -> Windows/macOS/Linux wheels + Consolidate + # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc + # both -> everything (previous default behaviour) + # Scheduled (daily) builds use this default too, so they build mssql-python only. + - name: buildPackage + displayName: 'Package to Build' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + - 'both' + default: 'mssql-python' # ========================= # PLATFORM CONFIGURATIONS @@ -384,16 +400,18 @@ extends: # 5. Builds wheel # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning - - ${{ each config in parameters.windowsConfigs }}: - - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self - parameters: - stageName: Win_py${{ config.pyVer }}_${{ config.arch }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python stages unless an odbc-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.windowsConfigs }}: + - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self + parameters: + stageName: Win_py${{ config.pyVer }}_${{ config.arch }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # MACOS BUILD STAGES @@ -410,15 +428,16 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ each config in parameters.macosConfigs }}: - - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self - parameters: - stageName: MacOS_py${{ config.pyVer }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.macosConfigs }}: + - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self + parameters: + stageName: MacOS_py${{ config.pyVer }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # LINUX BUILD STAGES @@ -440,15 +459,16 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ each config in parameters.linuxConfigs }}: - - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self - parameters: - stageName: Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheels - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.linuxConfigs }}: + - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self + parameters: + stageName: Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheels + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # CONSOLIDATE STAGE @@ -462,36 +482,37 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - stage: Consolidate - displayName: 'Consolidate All Artifacts' - dependsOn: - # Windows dependencies (9 stages) - - Win_py310_x64 - - Win_py311_x64 - - Win_py312_x64 - - Win_py313_x64 - - Win_py314_x64 - - Win_py311_arm64 - - Win_py312_arm64 - - Win_py313_arm64 - - Win_py314_arm64 - # macOS dependencies (5 stages) - - MacOS_py310 - - MacOS_py311 - - MacOS_py312 - - MacOS_py313 - - MacOS_py314 - # Linux dependencies (4 stages) - - Linux_manylinux_2_28_x86_64 - - Linux_manylinux_2_28_aarch64 - - Linux_musllinux_x86_64 - - Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self - parameters: - # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' - # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - stage: Consolidate + displayName: 'Consolidate All Artifacts' + dependsOn: + # Windows dependencies (9 stages) + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + - Win_py311_arm64 + - Win_py312_arm64 + - Win_py313_arm64 + - Win_py314_arm64 + # macOS dependencies (5 stages) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # Linux dependencies (4 stages) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + - Linux_musllinux_x86_64 + - Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self + parameters: + # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' + # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') + oneBranchType: '${{ variables.effectiveOneBranchType }}' # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging @@ -507,32 +528,36 @@ extends: # first-party native code is introduced. # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================================================================= # CONSOLIDATE ODBC STAGE @@ -541,18 +566,19 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - stage: ConsolidateOdbc - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 From 9d3097c8331d5caed76a8ade8a2d28baded2bdba Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:21:13 +0530 Subject: [PATCH 13/35] build: nightly build always builds both packages Add effectiveBuildPackage variable that forces 'both' on scheduled (Build.Reason=Schedule) runs while manual/PR runs honour the buildPackage parameter. Mirrors the effectiveOneBranchType pattern so the nightly keeps refreshing every artifact. --- .../build-release-package-pipeline.yml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 29e735080..c485cc8ee 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -76,7 +76,8 @@ parameters: # mssql-python -> Windows/macOS/Linux wheels + Consolidate # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc # both -> everything (previous default behaviour) - # Scheduled (daily) builds use this default too, so they build mssql-python only. + # Scheduled (daily) builds ignore this and always build 'both' (see + # effectiveBuildPackage below) so the nightly keeps producing every artifact. - name: buildPackage displayName: 'Package to Build' type: string @@ -192,6 +193,14 @@ variables: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' + + # Determine which package(s) to build: scheduled (nightly) builds always build + # 'both' so every artifact keeps refreshing; manual/PR builds honour the parameter. + - name: effectiveBuildPackage + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'both' + ${{ else }}: + value: '${{ parameters.buildPackage }}' # Variable template imports # Each file provides specific variable groups: @@ -401,7 +410,7 @@ extends: # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning # Gate: build mssql-python stages unless an odbc-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.windowsConfigs }}: - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self parameters: @@ -428,7 +437,7 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.macosConfigs }}: - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self parameters: @@ -459,7 +468,7 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.linuxConfigs }}: - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self parameters: @@ -482,7 +491,7 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - stage: Consolidate displayName: 'Consolidate All Artifacts' dependsOn: @@ -529,7 +538,7 @@ extends: # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcWindowsConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self parameters: @@ -539,7 +548,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcMacosConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self parameters: @@ -548,7 +557,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcLinuxConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self parameters: @@ -566,7 +575,7 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - stage: ConsolidateOdbc displayName: 'Consolidate All ODBC Artifacts' dependsOn: From fdb02cb62f91c3280f2d2cbf834ddb97793de7ee Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 20:27:59 +0530 Subject: [PATCH 14/35] Add OneBranch release pipelines for mssql-python-odbc Adds the build and release pipelines for the standalone mssql-python-odbc package: per-platform build stages (Windows x64/arm64, macOS universal2, Linux manylinux_2_28 + musllinux_1_2 for x86_64/aarch64), a consolidate job that gathers all 7 py3-none wheels, the build orchestration pipeline, and the ESRP-based official release pipeline. The build stages run no compilation (data-only package); Linux builds run in tdslibrs ACR PyPA containers and the musllinux tag is auto-detected via libc_ver. Infra follow-ups (ESRP onboarding of the mssql-python-odbc name, registering the ADO build definition to set odbcBuildDefinitionId, and sharing the ESRP variable group) are marked with TODO(infra) comments. --- .../build-release-odbc-pipeline.yml | 210 ++++++++++++++++ .../jobs/consolidate-odbc-artifacts-job.yml | 68 ++++++ .../official-release-odbc-pipeline.yml | 227 ++++++++++++++++++ .../stages/build-odbc-linux-stage.yml | 163 +++++++++++++ .../stages/build-odbc-macos-stage.yml | 107 +++++++++ .../stages/build-odbc-windows-stage.yml | 125 ++++++++++ 6 files changed, 900 insertions(+) create mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml create mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml create mode 100644 OneBranchPipelines/stages/build-odbc-linux-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-macos-stage.yml create mode 100644 OneBranchPipelines/stages/build-odbc-windows-stage.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml new file mode 100644 index 000000000..260fd1c3c --- /dev/null +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -0,0 +1,210 @@ +# ========================================================================================= +# OneBranch Build Pipeline for mssql-python-odbc +# ========================================================================================= +# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. +# +# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is +# NO compiled Python extension, so every wheel is tagged `py3-none-` and a +# SINGLE build per platform covers all supported Python versions (3.10+). That is why +# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces +# only 7 wheels total: +# - Windows: x64, arm64 +# - macOS: universal2 +# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) +# +# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml +# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). +# +# INFRA PREREQUISITES (see companion release pipeline for the definition ID): +# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). +# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. +# - Ensure the reused pools/images are available: Python-1ES-pool +# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the +# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. +# ========================================================================================= + +# Build number format: YYDDD.r +name: $(Year:YY)$(DayOfYear)$(Rev:.r) + +# Only trigger when ODBC-package inputs change (path filters are literal prefixes, +# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. +trigger: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +pr: + branches: + include: + - main + paths: + include: + - setup_odbc.py + - mssql_python_odbc + - mssql_python/libs + - OneBranchPipelines/build-release-odbc-pipeline.yml + - OneBranchPipelines/stages/build-odbc-windows-stage.yml + - OneBranchPipelines/stages/build-odbc-macos-stage.yml + - OneBranchPipelines/stages/build-odbc-linux-stage.yml + - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml + +parameters: + - name: oneBranchType + displayName: 'OneBranch Template Type' + type: string + values: + - 'Official' + - 'NonOfficial' + default: 'NonOfficial' + + - name: runSdlTasks + displayName: 'Run SDL Security Tasks' + type: boolean + default: true + + # Windows: one wheel per architecture (NO per-Python matrix). + - name: odbcWindowsConfigs + type: object + default: + - arch: 'x64' + - arch: 'arm64' + + # macOS: single universal2 wheel. + - name: odbcMacosConfigs + type: object + default: + - name: 'universal2' + + # Linux: one wheel per libc/arch combo. + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + +variables: + # Scheduled builds run as Official; manual/PR builds use the parameter. + - name: effectiveOneBranchType + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'Official' + ${{ else }}: + value: '${{ parameters.oneBranchType }}' + + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the + # per-stage EsrpMalwareScanning tasks. + - template: /OneBranchPipelines/variables/signing-variables.yml@self + - group: 'ESRP Federated Creds (AME)' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + +extends: + template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + + # This pipeline compiles NO first-party native code — it only packages the + # pre-built ODBC driver binaries, which are signed and governed by the Microsoft + # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; + # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). + binskim: + enabled: false + justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' + apiscan: + enabled: false + justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' + armory: + enabled: false + codeql: + enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' + + credscan: + enabled: ${{ parameters.runSdlTasks }} + policheck: + enabled: ${{ parameters.runSdlTasks }} + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: ${{ parameters.runSdlTasks }} + sbom: + enabled: ${{ parameters.runSdlTasks }} + packageName: 'mssql-python-odbc' + tsa: + enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + # WINDOWS BUILD STAGES (2: x64, arm64) + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # MACOS BUILD STAGE (1: universal2) + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # CONSOLIDATE STAGE (waits for all 7 platform stages) + - stage: Consolidate + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml new file mode 100644 index 000000000..8ac829631 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -0,0 +1,68 @@ +# Consolidate ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-odbc` wheels from all build stages and +# consolidates them into a single dist/ folder for the release pipeline to publish. +# Expected: 7 wheels (2 Windows + 1 macOS universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download ALL artifacts from the current build (one per platform stage). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All ODBC Platform Artifacts' + inputs: + buildType: 'current' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" + else + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + fi + displayName: 'Consolidate ODBC wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_Consolidate_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml new file mode 100644 index 000000000..3a9898c25 --- /dev/null +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -0,0 +1,227 @@ +# ========================================================================================= +# OneBranch Official Release Pipeline for mssql-python-odbc +# ========================================================================================= +# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to +# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group +# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python +# release (same inputs and FolderLocation); the only differences are the build pipeline +# source and that this package has no debug symbols to publish. +# +# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the +# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# ========================================================================================= + +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC + +# Manual trigger only — releases should be deliberate. +trigger: none +pr: none + +parameters: + - name: releaseToPyPI + displayName: 'Release to PyPI (Production)' + type: boolean + default: false # Safety: default false to prevent accidental releases. + +variables: + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # Shared ESRP identity — same group used by the mssql-python release pipeline. + - group: 'ESRP Federated Creds (AME)' + # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # once it is registered (the mssql-python build definition is 2199). Left as 0 so the + # pipeline compiles; the download tasks will fail at runtime until this is set. + - name: odbcBuildDefinitionId + value: '0' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + + # Reference to the ODBC build pipeline (select a run in the UI when releasing). + pipelines: + - pipeline: buildPipeline + source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + trigger: none + +extends: + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' + + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress + suppressionSet: default + + # Artifacts were already scanned during the build; this pipeline only republishes. + binskim: + enabled: false + justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + tsa: + enabled: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + - stage: ReleasePackages + displayName: 'Release mssql-python-odbc to PyPI' + + jobs: + # Job 1: Download + validate the consolidated wheels (custom pool). + - job: PrepAndValidate + displayName: 'Download and Validate Artifacts' + + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Consolidated ODBC Artifacts' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + - task: PowerShell@2 + displayName: 'List and Verify Downloaded Wheels' + inputs: + targetType: 'inline' + script: | + $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" + if (-not (Test-Path $wheelsPath)) { + Write-Error "Wheel directory not found at: $wheelsPath" + exit 1 + } + $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" + Write-Host "Total wheel files found: $($wheels.Count)" + + $allValid = $true + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($wheel in $wheels) { + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) + $entryCount = $zip.Entries.Count + $zip.Dispose() + Write-Host " OK $($wheel.Name) ($entryCount entries)" + } catch { + Write-Error " BAD $($wheel.Name): $_" + $allValid = $false + } + } + if (-not $allValid) { + Write-Error "One or more wheel files are corrupted" + exit 1 + } + if ($wheels.Count -ne 7) { + Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + } + Write-Host "All wheels verified successfully!" + + # Dry-run summary (when release is disabled). + - ${{ if eq(parameters.releaseToPyPI, false) }}: + - task: PowerShell@2 + displayName: 'Dry Run - Release Skipped' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "DRY RUN MODE - No Release Performed" + Write-Host "Package: mssql-python-odbc" + Write-Host "====================================" + Write-Host "To perform an actual release:" + Write-Host "1. Set 'releaseToPyPI' parameter to true" + Write-Host "2. Re-run pipeline" + + # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). + # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. + - ${{ if eq(parameters.releaseToPyPI, true) }}: + - job: PyPIRelease + displayName: 'ESRP Release to PyPI' + dependsOn: PrepAndValidate + + templateContext: + type: releaseJob + isProduction: true + + pool: + type: windows + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' + + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Artifacts from ODBC Build Pipeline' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: $(odbcBuildDefinitionId) + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts' + + # ESRP release task — identical to the mssql-python release pipeline + # (same shared ESRP identity and PyPI publish inputs). + - task: EsrpRelease@9 + displayName: 'ESRP Release to PyPI' + inputs: + connectedservicename: '$(ESRPConnectedServiceName)' + usemanagedidentity: true + keyvaultname: '$(AuthAKVName)' + signcertname: '$(AuthSignCertName)' + clientid: '$(EsrpClientId)' + Intent: 'PackageDistribution' + ContentType: 'PyPI' + ContentSource: 'Folder' + FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' + WaitForReleaseCompletion: true + Owners: '$(owner)' + Approvers: '$(approver)' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'ESRPRELPACMAN' + DomainTenantId: '$(DomainTenantId)' + + - task: PowerShell@2 + displayName: 'Release Summary' + inputs: + targetType: 'inline' + script: | + Write-Host "====================================" + Write-Host "ESRP Release Completed" + Write-Host "Package: mssql-python-odbc" + Write-Host "Target: PyPI" + Write-Host "====================================" + Write-Host "Next steps:" + Write-Host "1. Verify release in the ESRP portal" + Write-Host "2. Wait for approval workflow completion" + Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml new file mode 100644 index 000000000..798b3c341 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -0,0 +1,163 @@ +# ODBC Linux Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE libc/arch combo +# (manylinux_2_28 or musllinux_1_2 x x86_64 or aarch64). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# A PyPA build container is still used so `platform.libc_ver()` reports the right +# libc and the wheel is tagged manylinux_2_28_* or musllinux_1_2_* correctly. +# aarch64 wheels are produced under QEMU emulation. Driver binaries come from the +# committed mssql_python/libs/ tree that is bind-mounted into the container. +parameters: + # Stage identifier (e.g., 'ODBC_Linux_manylinux_2_28_x86_64'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + # Distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc). + - name: linuxTag + type: string + # CPU architecture: 'x86_64' (AMD64) or 'aarch64' (ARM64). + - name: arch + type: string + # Docker platform for QEMU emulation: 'linux/amd64' or 'linux/arm64'. + - name: dockerPlatform + type: string + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Reuse the same custom 1ES Linux pool as the mssql-python build. + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 60 + + variables: + # BinSkim needs ICU libs not present in manylinux/musllinux containers. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + - bash: | + set -e + if ! docker info > /dev/null 2>&1; then + echo "Starting Docker daemon..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Setup and start Docker daemon' + + - script: | + sudo apt-get install -y qemu-user-static + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(ob_outputDirectory)/wheels + mkdir -p $(ob_outputDirectory)/wheels + displayName: 'Prepare artifact directories' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + # Same PyPA images as the mssql-python Linux build (mirrored into ACR). + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'. Expected: manylinux_2_28 or musllinux" >&2 + exit 1 + fi + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $IMAGE tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + # Build the single py3-none wheel inside the container. Any CPython works; no + # compilation runs. setup_odbc.py's get_platform_info() auto-detects musl via + # platform.libc_ver(): on musllinux images it ALWAYS emits musllinux_1_2_ + # and ignores MANYLINUX_TAG. MANYLINUX_TAG is only consulted on glibc images, so + # hardcoding "manylinux_2_28" below is correct for both image families. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + docker exec \ + -e targetArch="$(ARCH)" \ + -e MANYLINUX_TAG="manylinux_2_28" \ + build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -eux + # Pick any available CPython from the PyPA image. + PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) + echo "Using: $($PY --version)" + $PY -m pip install -q -U pip setuptools wheel + cd /workspace + $PY setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh /workspace/dist/*.whl + ' + displayName: 'Build ODBC wheel in container' + + - script: | + set -euxo pipefail + cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Linux Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml new file mode 100644 index 000000000..9b47388e3 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -0,0 +1,107 @@ +# ODBC macOS Single Configuration Stage Template +# Builds the universal2 `mssql-python-odbc` wheel (arm64 + x86_64 driver binaries in +# one wheel). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# `setup_odbc.py` returns the macosx_15_0_universal2 platform tag and copies both +# macOS arch subtrees from mssql_python/libs, so a single build serves both slices. +parameters: + # Stage identifier (e.g., 'ODBC_MacOS_universal2'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + - name: oneBranchType + type: string + default: 'Official' + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC macOS Universal2' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - macOS Universal2' + # macOS pools declare as 'linux' type (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-14' + timeoutInMinutes: 60 + + variables: + # BinSkim targets PE binaries; macOS uses Mach-O, so disable it here. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS stages). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - script: | + set -e + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # get_platform_info() forces the macosx_15_0_universal2 tag; sync_libs() copies + # both arm64 + x86_64 driver subtrees, so no post-build retag is required. + - script: | + set -e + python setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh dist/*.whl + displayName: 'Build ODBC wheel (universal2)' + + - script: | + set -e + python -m twine check dist/*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)/wheels' + displayName: 'Stage wheel artifact' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC macOS Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml new file mode 100644 index 000000000..7a812de45 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -0,0 +1,125 @@ +# ODBC Windows Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE Windows architecture. +# +# Unlike the mssql-python build, this package ships ONLY pre-built ODBC driver +# binaries (data) — there is NO compiled Python extension. A single +# `py3-none-` wheel therefore serves every supported Python version +# (3.10+), so this stage does NOT use a per-Python-version matrix and does NOT +# run pytest against SQL Server. The driver binaries are already committed under +# `mssql_python/libs/`, so `checkout: self` is all that is needed; `setup_odbc.py` +# packages the current platform's subtree. +parameters: + # Stage identifier (e.g., 'ODBC_Win_x64'). + - name: stageName + type: string + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # Target architecture: 'x64' (AMD64) or 'arm64' (ARM64). + - name: architecture + type: string + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheel. + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Windows ${{ parameters.architecture }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - Windows ${{ parameters.architecture }}' + # Reuse the same custom 1ES Windows pool as the mssql-python build. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + # OneBranch output directory for artifacts. + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows stages). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). + targetArch: ${{ parameters.architecture }} + + steps: + # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + # The ARM64 wheel is pure data, so it can be produced on an x64 host — there is + # no native compilation and hence no ARM64 python.lib cross-compile machinery. + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build the platform-specific, Python-agnostic wheel. ARCHITECTURE controls the + # platform tag (win_amd64 / win_arm64). No native compilation happens here — + # setup_odbc.py only packages the driver binaries from mssql_python/libs. + - powershell: | + $ErrorActionPreference = "Stop" + $env:ARCHITECTURE = "${{ parameters.architecture }}" + python setup_odbc.py bdist_wheel + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + displayName: 'Build ODBC wheel (ARCHITECTURE=${{ parameters.architecture }})' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m twine check dist\*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifact' + + # OneBranch requires artifact naming: drop__. + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Windows Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheel for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 From 79d60ba3a693fc43c887e3d78fbc9fb7754e93b3 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:43:11 +0530 Subject: [PATCH 15/35] Address PR review: fail-fast on wheel count, armory justification, Linux twine check + container cleanup, reword infra TODOs --- OneBranchPipelines/build-release-odbc-pipeline.yml | 1 + .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++--- OneBranchPipelines/official-release-odbc-pipeline.yml | 9 +++++---- OneBranchPipelines/stages/build-odbc-linux-stage.yml | 8 +++++++- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml index 260fd1c3c..b64dc936b 100644 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ b/OneBranchPipelines/build-release-odbc-pipeline.yml @@ -144,6 +144,7 @@ extends: justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' armory: enabled: false + justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' codeql: enabled: false justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 8ac829631..0225d34a2 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -54,10 +54,10 @@ jobs: WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then - echo "WARNING: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" - else - echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." displayName: 'Consolidate ODBC wheels' - task: PublishPipelineArtifact@1 diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 3a9898c25..90eaedddb 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -8,7 +8,7 @@ # source and that this package has no debug symbols to publish. # # This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two TODO(infra) values below are set. +# `mssql-python-odbc` project name lands and the two infra values below are set. # ========================================================================================= name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC @@ -28,7 +28,7 @@ variables: - template: /OneBranchPipelines/variables/onebranch-variables.yml@self # Shared ESRP identity — same group used by the mssql-python release pipeline. - group: 'ESRP Federated Creds (AME)' - # TODO(infra): set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' + # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' # once it is registered (the mssql-python build definition is 2199). Left as 0 so the # pipeline compiles; the download tasks will fail at runtime until this is set. - name: odbcBuildDefinitionId @@ -44,7 +44,7 @@ resources: # Reference to the ODBC build pipeline (select a run in the UI when releasing). pipelines: - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # TODO(infra): match the registered build pipeline name. + source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. trigger: none extends: @@ -141,7 +141,8 @@ extends: exit 1 } if ($wheels.Count -ne 7) { - Write-Warning "Expected 7 ODBC wheels but found $($wheels.Count)" + Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" + exit 1 } Write-Host "All wheels verified successfully!" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index 798b3c341..d0a9883cd 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -123,9 +123,10 @@ stages: # Pick any available CPython from the PyPA image. PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) echo "Using: $($PY --version)" - $PY -m pip install -q -U pip setuptools wheel + $PY -m pip install -q -U pip setuptools wheel twine cd /workspace $PY setup_odbc.py bdist_wheel + $PY -m twine check /workspace/dist/*.whl echo "Produced wheels:"; ls -lh /workspace/dist/*.whl ' displayName: 'Build ODBC wheel in container' @@ -135,6 +136,11 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + - script: | + docker rm -f build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Stop and remove build container' + condition: always() + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Linux Artifact' inputs: From d4b82a613f8d51966c1e40a8449e82b37aa51f39 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 7 Jul 2026 21:58:20 +0530 Subject: [PATCH 16/35] Address review: preflight guard for odbcBuildDefinitionId placeholder; drop unused targetArch var on Windows stage --- OneBranchPipelines/official-release-odbc-pipeline.yml | 11 +++++++++++ .../stages/build-odbc-windows-stage.yml | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml index 90eaedddb..ef06b0895 100644 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ b/OneBranchPipelines/official-release-odbc-pipeline.yml @@ -99,6 +99,17 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' steps: + - task: PowerShell@2 + displayName: 'Preflight: verify odbcBuildDefinitionId is configured' + inputs: + targetType: 'inline' + script: | + if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { + Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." + exit 1 + } + Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" + - task: DownloadPipelineArtifact@2 displayName: 'Download Consolidated ODBC Artifacts' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 7a812de45..57de838ed 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -48,8 +48,6 @@ stages: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' # OneBranch-required variable (unused on Windows stages). LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' - # Drives the wheel platform tag in setup_odbc.py (get_platform_info()). - targetArch: ${{ parameters.architecture }} steps: # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. From 68b7ba00d98ec1cd30ef7ae6ba4925d25edc8a80 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:33:46 +0530 Subject: [PATCH 17/35] Fold mssql-python-odbc build+release into existing pipelines Per review feedback, build and release the standalone mssql-python-odbc package from the existing Build-Release-Package-Pipeline (def 2199) and official release pipeline instead of separate ADO definitions. Build pipeline: add odbcWindows/Macos/LinuxConfigs params (2+1+4=7 data-only wheels), reference the 3 odbc stage templates alongside the mssql-python stages, and add a separate ConsolidateOdbc stage publishing drop_ConsolidateOdbc_ConsolidateArtifacts (distinct from mssql-python's drop_Consolidate_ConsolidateArtifacts). Release pipeline: add releasePackage parameter (mssql-python | mssql-python-odbc); select the consolidated artifact accordingly and skip symbol publishing + mssql-py-core version validation for odbc. Delete the redundant standalone orchestrators (build-release-odbc-pipeline.yml, official-release-odbc-pipeline.yml). This removes the need to register a new ADO build definition and the odbcBuildDefinitionId placeholder. Both packages now build from def 2199; the odbc stages consume setup_odbc.py from PR #663, so #663 must merge before these stages can build. --- .../build-release-odbc-pipeline.yml | 211 ---------------- .../build-release-package-pipeline.yml | 94 ++++++- .../jobs/consolidate-odbc-artifacts-job.yml | 5 +- .../official-release-odbc-pipeline.yml | 239 ------------------ .../official-release-pipeline.yml | 56 ++-- 5 files changed, 136 insertions(+), 469 deletions(-) delete mode 100644 OneBranchPipelines/build-release-odbc-pipeline.yml delete mode 100644 OneBranchPipelines/official-release-odbc-pipeline.yml diff --git a/OneBranchPipelines/build-release-odbc-pipeline.yml b/OneBranchPipelines/build-release-odbc-pipeline.yml deleted file mode 100644 index b64dc936b..000000000 --- a/OneBranchPipelines/build-release-odbc-pipeline.yml +++ /dev/null @@ -1,211 +0,0 @@ -# ========================================================================================= -# OneBranch Build Pipeline for mssql-python-odbc -# ========================================================================================= -# Builds the standalone ODBC driver-binary package `mssql-python-odbc`. -# -# This package ships ONLY the Microsoft ODBC Driver 18 binaries as data. There is -# NO compiled Python extension, so every wheel is tagged `py3-none-` and a -# SINGLE build per platform covers all supported Python versions (3.10+). That is why -# this pipeline has NO per-Python-version matrix (unlike mssql-python) and produces -# only 7 wheels total: -# - Windows: x64, arm64 -# - macOS: universal2 -# - Linux: manylinux_2_28 (x86_64, aarch64), musllinux_1_2 (x86_64, aarch64) -# -# Publishing to PyPI is handled by the companion official-release-odbc-pipeline.yml -# using the SAME shared ESRP identity (variable group 'ESRP Federated Creds (AME)'). -# -# INFRA PREREQUISITES (see companion release pipeline for the definition ID): -# - Register this YAML as a new ADO build definition (e.g. 'Build-Release-ODBC-Pipeline'). -# - Share the 'ESRP Federated Creds (AME)' variable group with the new pipeline. -# - Ensure the reused pools/images are available: Python-1ES-pool -# (PYTHON-1ES-MMS2022 for Windows, PYTHON-1ES-UB2404 for Linux) and the -# 'Magnitude Test-mssql-rs-mssql-python' Azure subscription for ACR login. -# ========================================================================================= - -# Build number format: YYDDD.r -name: $(Year:YY)$(DayOfYear)$(Rev:.r) - -# Only trigger when ODBC-package inputs change (path filters are literal prefixes, -# not globs, in Azure DevOps). Unrelated mssql-python commits do not rebuild this. -trigger: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -pr: - branches: - include: - - main - paths: - include: - - setup_odbc.py - - mssql_python_odbc - - mssql_python/libs - - OneBranchPipelines/build-release-odbc-pipeline.yml - - OneBranchPipelines/stages/build-odbc-windows-stage.yml - - OneBranchPipelines/stages/build-odbc-macos-stage.yml - - OneBranchPipelines/stages/build-odbc-linux-stage.yml - - OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml - -parameters: - - name: oneBranchType - displayName: 'OneBranch Template Type' - type: string - values: - - 'Official' - - 'NonOfficial' - default: 'NonOfficial' - - - name: runSdlTasks - displayName: 'Run SDL Security Tasks' - type: boolean - default: true - - # Windows: one wheel per architecture (NO per-Python matrix). - - name: odbcWindowsConfigs - type: object - default: - - arch: 'x64' - - arch: 'arm64' - - # macOS: single universal2 wheel. - - name: odbcMacosConfigs - type: object - default: - - name: 'universal2' - - # Linux: one wheel per libc/arch combo. - - name: odbcLinuxConfigs - type: object - default: - - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } - - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } - -variables: - # Scheduled builds run as Official; manual/PR builds use the parameter. - - name: effectiveOneBranchType - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - value: 'Official' - ${{ else }}: - value: '${{ parameters.oneBranchType }}' - - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Maps 'ESRP Federated Creds (AME)' group vars to the Signing* names used by the - # per-stage EsrpMalwareScanning tasks. - - template: /OneBranchPipelines/variables/signing-variables.yml@self - - group: 'ESRP Federated Creds (AME)' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - -extends: - template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress - suppressionSet: default - - # This pipeline compiles NO first-party native code — it only packages the - # pre-built ODBC driver binaries, which are signed and governed by the Microsoft - # ODBC Driver team's own SDL. Native/binary analyzers are therefore disabled here; - # every wheel is still malware-scanned per stage (EsrpMalwareScanning@5). - binskim: - enabled: false - justificationForDisabling: 'No first-party native code is built here; the packaged ODBC driver binaries are pre-built/pre-signed and covered by the ODBC Driver team SDL. Wheels are malware-scanned per stage.' - apiscan: - enabled: false - justificationForDisabling: 'No first-party native binaries are produced by this pipeline.' - armory: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party native code is built. Wheels are malware-scanned per stage.' - codeql: - enabled: false - justificationForDisabling: 'This pipeline only packages pre-built driver binaries; no first-party source is compiled.' - - credscan: - enabled: ${{ parameters.runSdlTasks }} - policheck: - enabled: ${{ parameters.runSdlTasks }} - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: ${{ parameters.runSdlTasks }} - sbom: - enabled: ${{ parameters.runSdlTasks }} - packageName: 'mssql-python-odbc' - tsa: - enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - # WINDOWS BUILD STAGES (2: x64, arm64) - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # MACOS BUILD STAGE (1: universal2) - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # LINUX BUILD STAGES (4: manylinux/musllinux x x86_64/aarch64) - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' - - # CONSOLIDATE STAGE (waits for all 7 platform stages) - - stage: Consolidate - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 3bd789e85..984e52bcc 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -138,6 +138,33 @@ parameters: - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= + # ODBC PACKAGE CONFIGURATIONS (mssql-python-odbc) + # ========================= + # The standalone mssql-python-odbc package ships ONLY pre-built ODBC driver + # binaries (data) — there is NO compiled extension — so ONE py3-none- + # wheel per platform serves every supported Python version. Hence there is no + # per-Python matrix here: 7 wheels total (2 Windows + 1 macOS + 4 Linux). + # These stages build alongside the mssql-python wheels and are consolidated + # SEPARATELY (see the ConsolidateOdbc stage) so the release pipeline can publish + # mssql-python and mssql-python-odbc independently. + - name: odbcWindowsConfigs + type: object + default: + - { arch: 'x64' } + - { arch: 'arm64' } + - name: odbcMacosConfigs + type: object + default: + - { name: 'universal2' } + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= # PIPELINE VARIABLES # ========================= @@ -338,8 +365,10 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # Total stages: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages - # Stages run in parallel (no dependencies between platform builds) + # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python-odbc: 2 Windows + 1 macOS + 4 Linux + 1 ConsolidateOdbc = 8 stages + # Total: 27 stages. All build stages run in parallel; each Consolidate* stage + # waits only on its own package's build stages. stages: # ========================= # WINDOWS BUILD STAGES @@ -466,3 +495,64 @@ extends: # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging + + # ========================================================================= + # ODBC PACKAGE BUILD STAGES (mssql-python-odbc) + # ========================================================================= + # 7 data-only wheels (no native compile, no pytest) built from setup_odbc.py. + # They run in parallel with the mssql-python stages and are consolidated by a + # SEPARATE ConsolidateOdbc stage so the release pipeline can ship each package + # independently. SDL "just works": these wheels contain the SAME ODBC driver + # binaries already scanned in mssql_python/libs by the main build — no new + # first-party native code is introduced. + # + # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC macOS stage (1): ODBC_MacOS_universal2 + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from + # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release + # pipeline selects one artifact based on its `releasePackage` parameter. + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 0225d34a2..42db132ff 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -64,5 +64,8 @@ jobs: displayName: 'Publish Consolidated ODBC Artifacts' inputs: targetPath: '$(ob_outputDirectory)' - artifact: 'drop_Consolidate_ConsolidateArtifacts' + # Distinct name so it does not collide with the mssql-python consolidate + # artifact (drop_Consolidate_ConsolidateArtifacts) in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateOdbc`. + artifact: 'drop_ConsolidateOdbc_ConsolidateArtifacts' publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-odbc-pipeline.yml b/OneBranchPipelines/official-release-odbc-pipeline.yml deleted file mode 100644 index ef06b0895..000000000 --- a/OneBranchPipelines/official-release-odbc-pipeline.yml +++ /dev/null @@ -1,239 +0,0 @@ -# ========================================================================================= -# OneBranch Official Release Pipeline for mssql-python-odbc -# ========================================================================================= -# Downloads the consolidated wheels from the ODBC build pipeline and publishes them to -# PyPI via ESRP, using the SAME shared ESRP identity as mssql-python (variable group -# 'ESRP Federated Creds (AME)'). The EsrpRelease task is identical to the mssql-python -# release (same inputs and FolderLocation); the only differences are the build pipeline -# source and that this package has no debug symbols to publish. -# -# This pipeline is ALWAYS Official. It is runnable the moment ESRP onboarding of the -# `mssql-python-odbc` project name lands and the two infra values below are set. -# ========================================================================================= - -name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release-ODBC - -# Manual trigger only — releases should be deliberate. -trigger: none -pr: none - -parameters: - - name: releaseToPyPI - displayName: 'Release to PyPI (Production)' - type: boolean - default: false # Safety: default false to prevent accidental releases. - -variables: - - template: /OneBranchPipelines/variables/common-variables.yml@self - - template: /OneBranchPipelines/variables/onebranch-variables.yml@self - # Shared ESRP identity — same group used by the mssql-python release pipeline. - - group: 'ESRP Federated Creds (AME)' - # Infra setup: set to the ADO build definition ID of 'Build-Release-ODBC-Pipeline' - # once it is registered (the mssql-python build definition is 2199). Left as 0 so the - # pipeline compiles; the download tasks will fail at runtime until this is set. - - name: odbcBuildDefinitionId - value: '0' - -resources: - repositories: - - repository: templates - type: git - name: 'OneBranch.Pipelines/GovernedTemplates' - ref: 'refs/heads/main' - - # Reference to the ODBC build pipeline (select a run in the UI when releasing). - pipelines: - - pipeline: buildPipeline - source: 'Build-Release-ODBC-Pipeline' # Infra setup: match the registered build pipeline name. - trigger: none - -extends: - template: 'v2/OneBranch.Official.CrossPlat.yml@templates' - - parameters: - featureFlags: - WindowsHostVersion: - Version: '2022' - - globalSdl: - baseline: - baselineFile: $(Build.SourcesDirectory)\.gdn\.gdnbaselines - suppressionSet: default - suppression: - suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress - suppressionSet: default - - # Artifacts were already scanned during the build; this pipeline only republishes. - binskim: - enabled: false - justificationForDisabling: 'Release pipeline only republishes pre-scanned wheels; no binaries are built here.' - credscan: - enabled: true - policheck: - enabled: true - break: true - exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' - publishLogs: - enabled: true - tsa: - enabled: true - configFile: '$(REPO_ROOT)/.config/tsaoptions.json' - - stages: - - stage: ReleasePackages - displayName: 'Release mssql-python-odbc to PyPI' - - jobs: - # Job 1: Download + validate the consolidated wheels (custom pool). - - job: PrepAndValidate - displayName: 'Download and Validate Artifacts' - - pool: - type: windows - isCustom: true - name: Python-1ES-pool - demands: - - imageOverride -equals PYTHON-1ES-MMS2022 - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - - steps: - - task: PowerShell@2 - displayName: 'Preflight: verify odbcBuildDefinitionId is configured' - inputs: - targetType: 'inline' - script: | - if ('$(odbcBuildDefinitionId)' -eq '0' -or [string]::IsNullOrWhiteSpace('$(odbcBuildDefinitionId)')) { - Write-Error "odbcBuildDefinitionId is still the '0' placeholder. Set it to the registered 'Build-Release-ODBC-Pipeline' build definition ID before releasing, otherwise the artifact download below fails cryptically." - exit 1 - } - Write-Host "odbcBuildDefinitionId = $(odbcBuildDefinitionId)" - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Consolidated ODBC Artifacts' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - - task: PowerShell@2 - displayName: 'List and Verify Downloaded Wheels' - inputs: - targetType: 'inline' - script: | - $wheelsPath = "$(Build.SourcesDirectory)/artifacts/dist" - if (-not (Test-Path $wheelsPath)) { - Write-Error "Wheel directory not found at: $wheelsPath" - exit 1 - } - $wheels = Get-ChildItem -Path $wheelsPath -Filter "*.whl" - Write-Host "Total wheel files found: $($wheels.Count)" - - $allValid = $true - Add-Type -AssemblyName System.IO.Compression.FileSystem - foreach ($wheel in $wheels) { - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel.FullName) - $entryCount = $zip.Entries.Count - $zip.Dispose() - Write-Host " OK $($wheel.Name) ($entryCount entries)" - } catch { - Write-Error " BAD $($wheel.Name): $_" - $allValid = $false - } - } - if (-not $allValid) { - Write-Error "One or more wheel files are corrupted" - exit 1 - } - if ($wheels.Count -ne 7) { - Write-Error "Expected 7 ODBC wheels but found $($wheels.Count)" - exit 1 - } - Write-Host "All wheels verified successfully!" - - # Dry-run summary (when release is disabled). - - ${{ if eq(parameters.releaseToPyPI, false) }}: - - task: PowerShell@2 - displayName: 'Dry Run - Release Skipped' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "DRY RUN MODE - No Release Performed" - Write-Host "Package: mssql-python-odbc" - Write-Host "====================================" - Write-Host "To perform an actual release:" - Write-Host "1. Set 'releaseToPyPI' parameter to true" - Write-Host "2. Re-run pipeline" - - # Job 2: ESRP Release (releaseJob on 1ES hosted pool — required by OneBranch policy). - # EsrpRelease is not allowed in custom pools; must use templateContext.type: releaseJob. - - ${{ if eq(parameters.releaseToPyPI, true) }}: - - job: PyPIRelease - displayName: 'ESRP Release to PyPI' - dependsOn: PrepAndValidate - - templateContext: - type: releaseJob - isProduction: true - - pool: - type: windows - - variables: - ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' - - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download Artifacts from ODBC Build Pipeline' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - definition: $(odbcBuildDefinitionId) - buildVersionToDownload: 'specific' - buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' - targetPath: '$(Build.SourcesDirectory)/artifacts' - - # ESRP release task — identical to the mssql-python release pipeline - # (same shared ESRP identity and PyPI publish inputs). - - task: EsrpRelease@9 - displayName: 'ESRP Release to PyPI' - inputs: - connectedservicename: '$(ESRPConnectedServiceName)' - usemanagedidentity: true - keyvaultname: '$(AuthAKVName)' - signcertname: '$(AuthSignCertName)' - clientid: '$(EsrpClientId)' - Intent: 'PackageDistribution' - ContentType: 'PyPI' - ContentSource: 'Folder' - FolderLocation: '$(Build.SourcesDirectory)/artifacts/dist' - WaitForReleaseCompletion: true - Owners: '$(owner)' - Approvers: '$(approver)' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMAN' - DomainTenantId: '$(DomainTenantId)' - - - task: PowerShell@2 - displayName: 'Release Summary' - inputs: - targetType: 'inline' - script: | - Write-Host "====================================" - Write-Host "ESRP Release Completed" - Write-Host "Package: mssql-python-odbc" - Write-Host "Target: PyPI" - Write-Host "====================================" - Write-Host "Next steps:" - Write-Host "1. Verify release in the ESRP portal" - Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify on PyPI: https://pypi.org/project/mssql-python-odbc/" diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index a5ec227f6..731df538e 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -1,5 +1,7 @@ -# OneBranch Official Release Pipeline for mssql-python -# Downloads wheel and symbol artifacts from build pipeline, publishes symbols, and releases wheels to PyPI via ESRP +# OneBranch Official Release Pipeline (mssql-python and mssql-python-odbc) +# Selects the package via the `releasePackage` parameter, downloads that package's +# consolidated wheels (and symbols for mssql-python) from the build pipeline +# (definition 2199), publishes symbols, and releases wheels to PyPI via ESRP. # This pipeline is ALWAYS Official - no NonOfficial option name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release @@ -10,6 +12,16 @@ pr: none # Parameters for release pipeline parameters: + # Which package to release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: 'Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: 'Publish Symbols to Symbol Servers' type: boolean @@ -30,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download/publish based on the target + # package. Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -117,7 +139,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -173,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -215,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -231,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "DRY RUN MODE - No Release Performed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "" Write-Host "Actions performed:" Write-Host "- Downloaded wheels from build pipeline" @@ -273,7 +297,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' - task: EsrpRelease@9 @@ -303,7 +327,7 @@ extends: Write-Host "====================================" Write-Host "ESRP Release Completed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" @@ -313,6 +337,6 @@ extends: Write-Host "Next steps:" Write-Host "1. Verify release in ESRP portal" Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify package on PyPI: https://pypi.org/project/mssql-python/" + Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" From 7a1d351c9a402c40e6c6d68e2d942770f2a20df2 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 11:50:04 +0530 Subject: [PATCH 18/35] Scope consolidate downloads by package to prevent cross-contamination Def 2199 now builds mssql-python and mssql-python-odbc in one run. Both consolidate jobs used DownloadPipelineArtifact@2 with buildType: current, which downloads every artifact in the run and copies all *.whl into dist/. In the merged run this caused: (1) the main Consolidate to sweep the 7 odbc wheels into the mssql-python release (count is warning-only, so silent), and (2) ConsolidateOdbc to see 34 wheels while expecting 7 and hard-fail. Add itemPattern to scope each download to its own package's build-stage artifacts (main: drop_Win_*/drop_MacOS_*/drop_Linux_*; odbc: drop_ODBC_*), giving clean partitioning with no wheel leakage. --- .../jobs/consolidate-artifacts-job.yml | 13 ++++++++++--- .../jobs/consolidate-odbc-artifacts-job.yml | 6 +++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 478803aa3..336c736a5 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -29,13 +29,20 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from current build - # Matrix jobs publish as: Windows_, macOS_, Linux_ - # This downloads all of them automatically (27 total artifacts) + # Download only the mssql-python platform-stage artifacts from the current build. + # The build definition now ALSO produces mssql-python-odbc artifacts (drop_ODBC_*) + # in the SAME run, so we scope this download with itemPattern to the mssql-python + # stages (drop_Win_*, drop_MacOS_*, drop_Linux_*). Without this, the odbc wheels + # would be swept into this package's dist/ and shipped to the mssql-python PyPI + # project. The odbc wheels are consolidated separately (ConsolidateOdbc stage). - task: DownloadPipelineArtifact@2 displayName: 'Download All Platform Artifacts' inputs: buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** targetPath: '$(Pipeline.Workspace)/all-artifacts' # Consolidate all wheels into single dist/ directory diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index 42db132ff..f0d8a0715 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -32,11 +32,15 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from the current build (one per platform stage). + # Download only the mssql-python-odbc platform-stage artifacts (drop_ODBC_*) from + # the current build. The same run also builds the mssql-python wheels (drop_Win_*, + # drop_MacOS_*, drop_Linux_*); itemPattern scopes this download to the odbc stages + # so ONLY the 7 odbc wheels are consolidated here (not the mssql-python wheels). - task: DownloadPipelineArtifact@2 displayName: 'Download All ODBC Platform Artifacts' inputs: buildType: 'current' + itemPattern: 'drop_ODBC_*/**' targetPath: '$(Pipeline.Workspace)/all-artifacts' - bash: | From 7e5947ea5d97c3dfeeb349813c5540e8e6ec48ce Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:16:59 +0530 Subject: [PATCH 19/35] Correct stale mssql-python wheel count in consolidate diagnostic (27 -> 34) The matrix is 9 Windows + 5 macOS + 4 Linux stages x 5 Python versions (cp310-cp314) = 34 wheels across 18 stages. The diagnostic said 27 (7+4+16), stale from an older 3.10-3.13 / 7-Windows matrix. Count check remains warning-only (no behavior change). --- .../jobs/consolidate-artifacts-job.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 336c736a5..52eb43381 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (15 stages total: 7 Windows + 4 macOS + 4 Linux) -# Each Linux stage builds 4 Python versions, resulting in 27 total wheels +# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) +# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels parameters: - name: oneBranchType type: string @@ -75,12 +75,12 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 27 wheels (7 Windows + 4 macOS + 16 Linux)" + echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" - if [ "$WHEEL_COUNT" -ne 27 ]; then - echo "WARNING: Expected 27 wheels but found $WHEEL_COUNT" + if [ "$WHEEL_COUNT" -ne 34 ]; then + echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" else - echo "SUCCESS: All 27 wheels consolidated!" + echo "SUCCESS: All 34 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' From 68cba53970bc35a82bab3c28786fe36b88162ade Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 13:49:09 +0530 Subject: [PATCH 20/35] Harden odbc release path: pin guard + wheel-content assertions official-release-pipeline.yml: gate 'Published symbols' dry-run/summary text to mssql-python so odbc runs no longer claim symbol publishing that never happens; add a mssql-python-only step asserting the wheel metadata pins mssql-python-odbc==18.6.0 (fails by design if released from a pre-pin build). build-odbc-{linux,macos,windows}-stage.yml: assert each built wheel actually contains the ODBC driver binary (msodbcsql*), guarding against the .gitignore '*.so' trap / silent sync_libs() skip shipping an empty data wheel that still passes twine check. --- .../official-release-pipeline.yml | 48 ++++++++++++++++++- .../stages/build-odbc-linux-stage.yml | 22 +++++++++ .../stages/build-odbc-macos-stage.yml | 21 ++++++++ .../stages/build-odbc-windows-stage.yml | 22 +++++++++ 4 files changed, 111 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 731df538e..7cb529796 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: 'Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "- Downloaded wheels from build pipeline" Write-Host "- Verified wheel integrity" Write-Host "- Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "- Published symbols to SqlClientDrivers org" } Write-Host "" @@ -331,7 +374,8 @@ extends: Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + $symbolsPublished = ("${{ parameters.publishSymbols }}" -eq "True") -and ("${{ parameters.releasePackage }}" -eq "mssql-python") + Write-Host "Symbols Published: $symbolsPublished" Write-Host "=====================================" Write-Host "" Write-Host "Next steps:" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index d0a9883cd..969c0b8b3 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -136,6 +136,28 @@ stages: cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ displayName: 'Stage wheel artifact' + # Guard against the .gitignore '*.so' trap or a silent sync_libs() skip + # shipping a data wheel with NO driver binary (twine check would still pass). + # The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python3 -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - script: | docker rm -f build-$(LINUX_TAG)-$(ARCH) || true displayName: 'Stop and remove build container' diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml index 9b47388e3..c1e7541b1 100644 --- a/OneBranchPipelines/stages/build-odbc-macos-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -79,6 +79,27 @@ stages: TargetFolder: '$(ob_outputDirectory)/wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + if python -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then + echo "OK: $whl contains the ODBC driver binary" + else + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + done + displayName: 'Assert wheel contains ODBC driver binary' + - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC macOS Artifact' inputs: diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index 57de838ed..dea809ed0 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -92,6 +92,28 @@ stages: TargetFolder: '$(ob_outputDirectory)\wheels' displayName: 'Stage wheel artifact' + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver DLL name contains 'msodbcsql' on every platform. + - powershell: | + $ErrorActionPreference = "Stop" + $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No wheels found to verify in $(ob_outputDirectory)\wheels" + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($whl in $wheels) { + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $hasDriver = $zip.Entries | Where-Object { $_.FullName -match 'msodbcsql' } + $zip.Dispose() + if (-not $hasDriver) { + Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary" + } + displayName: 'Assert wheel contains ODBC driver binary' + # OneBranch requires artifact naming: drop__. - task: PublishPipelineArtifact@1 displayName: 'Publish ODBC Windows Artifact' From 0772cc0683c3624e93599ce94afb75d1438cd1c8 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Wed, 8 Jul 2026 14:24:25 +0530 Subject: [PATCH 21/35] Hard-fail mssql-python consolidate on wheel-count mismatch Flip the 34-wheel count check from WARNING-only to a hard exit 1, symmetric with the odbc consolidate job's count guard. A mismatch means a build stage silently produced a partial wheel set (e.g. a missing Python version) or the matrix changed without updating the expected count -- previously this shipped silently. --- OneBranchPipelines/jobs/consolidate-artifacts-job.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 52eb43381..40cda12b0 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -77,8 +77,13 @@ jobs: echo "Total wheel count: $WHEEL_COUNT" echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" + # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). + # A mismatch means a build stage silently produced a partial set (e.g. a missing + # Python version) or the build matrix changed without updating this count. + # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. if [ "$WHEEL_COUNT" -ne 34 ]; then - echo "WARNING: Expected 34 wheels but found $WHEEL_COUNT" + echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + exit 1 else echo "SUCCESS: All 34 wheels consolidated!" fi From 52dbd7bea53f3e9be6e95e1be909e05126d943d4 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 14:04:13 +0530 Subject: [PATCH 22/35] odbc: add mssql-python-odbc selector to dummy release pipeline Mirrors the official-release-pipeline odbc changes: releasePackage parameter, consolidatedArtifactName selection for both artifact downloads, and gating the mssql-py-core version validation + symbol publishing to mssql-python only. Stays on Maven ContentType so it never publishes to PyPI. --- OneBranchPipelines/dummy-release-pipeline.yml | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index f9e548fb0..28751e845 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,6 +12,16 @@ pr: none # Parameters for DUMMY release pipeline parameters: + # Which package to test-release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: '[TEST] Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: '[TEST] Publish Symbols to Symbol Servers' type: boolean @@ -32,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download based on the target package. + # Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -114,7 +134,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -175,13 +195,15 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: '[TEST] Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # the mssql-python-odbc package ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -217,8 +239,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (if enabled and symbols exist; mssql-python only) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -233,7 +255,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DRY RUN MODE ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "" Write-Host "Actions performed:" Write-Host "✓ Downloaded wheels from build pipeline" @@ -280,7 +302,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' # ⚠️ IMPORTANT: Uses Maven ContentType for testing - NOT PyPI! @@ -313,7 +335,7 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DUMMY RELEASE COMPLETED ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" From 19f1db171d59a9f8b49833ccb33c9ebbdfd20eaa Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:00:10 +0530 Subject: [PATCH 23/35] odbc: align dummy release pipeline mssql-python handling with official Adds the mssql-python-odbc pin guard (gated to mssql-python, matching official-release-pipeline) and gates the 'symbols published' summary messaging on releasePackage==mssql-python so the dummy pipeline exercises the exact same mssql-python validation/messaging as the official pipeline for both packages. --- OneBranchPipelines/dummy-release-pipeline.yml | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 28751e845..c98c351ff 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -196,7 +196,7 @@ extends: Write-Host "=====================================" # Step 3: Validate mssql-py-core is a stable version (mssql-python only; - # the mssql-python-odbc package ships no mssql-py-core.version file) + # mssql-python-odbc ships no mssql-py-core.version file) - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: - task: PowerShell@2 displayName: '[TEST] Validate mssql-py-core is a stable version' @@ -205,6 +205,49 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Guard: the released mssql-python wheel MUST depend on the pinned + # mssql-python-odbc package. Catches releasing from a PRE-pin build + # (e.g. the wrong build run), which would ship a driver-less package + # that breaks 'pip install mssql-python'. By design this FAILS on a + # pre-pin build. Update $expectedVersion when the odbc pin bumps. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.0' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if ($odbcLines) { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + } + $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + if ($metadata -notmatch $pinRegex) { + Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' @@ -261,7 +304,7 @@ extends: Write-Host "✓ Downloaded wheels from build pipeline" Write-Host "✓ Verified wheel integrity" Write-Host "✓ Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Published symbols to SqlClientDrivers org" } Write-Host "✗ ESRP dummy release NOT performed (parameter disabled)" @@ -339,7 +382,7 @@ extends: Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + Write-Host "Symbols Published: $(if ('${{ parameters.publishSymbols }}' -eq 'True' -and '${{ parameters.releasePackage }}' -eq 'mssql-python') { 'True' } else { 'False' })" Write-Host "=====================================" Write-Host "" Write-Host "⚠️ IMPORTANT: This was a DUMMY release using Maven ContentType" @@ -348,7 +391,7 @@ extends: Write-Host "What was tested:" Write-Host "✓ Artifact download from build pipeline" Write-Host "✓ Wheel integrity verification" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Symbol publishing to SqlClientDrivers org" } Write-Host "✓ ESRP release workflow (Maven ContentType)" From 2038ca63469e2bb17e83de6fefb6e1313be4e6b7 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:28:24 +0530 Subject: [PATCH 24/35] odbc: bump mssql-python-odbc pin to 18.6.2 in release pipelines --- OneBranchPipelines/dummy-release-pipeline.yml | 2 +- OneBranchPipelines/official-release-pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index c98c351ff..6cbb7c44b 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 7cb529796..8937fd0f6 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -216,7 +216,7 @@ extends: targetType: 'inline' script: | $ErrorActionPreference = 'Stop' - $expectedVersion = '18.6.0' + $expectedVersion = '18.6.2' $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" if ($wheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." From 926ba53e3ee266ed14b4e48acd45e48937032da2 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:17:36 +0530 Subject: [PATCH 25/35] build: add buildPackage selector to build pipeline Gate mssql-python and mssql-python-odbc stage groups behind a buildPackage parameter (mssql-python default) so only the selected package builds, halving run time and enabling concurrent per-package builds. Mirrors the releasePackage selector pattern in the release pipelines. --- .../build-release-package-pipeline.yml | 216 ++++++++++-------- 1 file changed, 121 insertions(+), 95 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 984e52bcc..29e735080 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -69,6 +69,22 @@ parameters: displayName: 'Run SDL Security Tasks' type: boolean default: true + + # Which package to build. Both are produced from this same pipeline but are + # fully independent stage groups, so building only one halves the run time and + # lets you run two builds (one per package) concurrently. + # mssql-python -> Windows/macOS/Linux wheels + Consolidate + # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc + # both -> everything (previous default behaviour) + # Scheduled (daily) builds use this default too, so they build mssql-python only. + - name: buildPackage + displayName: 'Package to Build' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + - 'both' + default: 'mssql-python' # ========================= # PLATFORM CONFIGURATIONS @@ -384,16 +400,18 @@ extends: # 5. Builds wheel # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning - - ${{ each config in parameters.windowsConfigs }}: - - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self - parameters: - stageName: Win_py${{ config.pyVer }}_${{ config.arch }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python stages unless an odbc-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.windowsConfigs }}: + - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self + parameters: + stageName: Win_py${{ config.pyVer }}_${{ config.arch }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # MACOS BUILD STAGES @@ -410,15 +428,16 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ each config in parameters.macosConfigs }}: - - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self - parameters: - stageName: MacOS_py${{ config.pyVer }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.macosConfigs }}: + - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self + parameters: + stageName: MacOS_py${{ config.pyVer }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # LINUX BUILD STAGES @@ -440,15 +459,16 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ each config in parameters.linuxConfigs }}: - - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self - parameters: - stageName: Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheels - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.linuxConfigs }}: + - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self + parameters: + stageName: Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheels + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # CONSOLIDATE STAGE @@ -462,36 +482,37 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - stage: Consolidate - displayName: 'Consolidate All Artifacts' - dependsOn: - # Windows dependencies (9 stages) - - Win_py310_x64 - - Win_py311_x64 - - Win_py312_x64 - - Win_py313_x64 - - Win_py314_x64 - - Win_py311_arm64 - - Win_py312_arm64 - - Win_py313_arm64 - - Win_py314_arm64 - # macOS dependencies (5 stages) - - MacOS_py310 - - MacOS_py311 - - MacOS_py312 - - MacOS_py313 - - MacOS_py314 - # Linux dependencies (4 stages) - - Linux_manylinux_2_28_x86_64 - - Linux_manylinux_2_28_aarch64 - - Linux_musllinux_x86_64 - - Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self - parameters: - # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' - # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - stage: Consolidate + displayName: 'Consolidate All Artifacts' + dependsOn: + # Windows dependencies (9 stages) + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + - Win_py311_arm64 + - Win_py312_arm64 + - Win_py313_arm64 + - Win_py314_arm64 + # macOS dependencies (5 stages) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # Linux dependencies (4 stages) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + - Linux_musllinux_x86_64 + - Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self + parameters: + # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' + # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') + oneBranchType: '${{ variables.effectiveOneBranchType }}' # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging @@ -507,32 +528,36 @@ extends: # first-party native code is introduced. # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 - - ${{ each config in parameters.odbcWindowsConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self - parameters: - stageName: ODBC_Win_${{ config.arch }} - jobName: BuildWheel - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ each config in parameters.odbcMacosConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self - parameters: - stageName: ODBC_MacOS_${{ config.name }} - jobName: BuildWheel - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ each config in parameters.odbcLinuxConfigs }}: - - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self - parameters: - stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheel - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================================================================= # CONSOLIDATE ODBC STAGE @@ -541,18 +566,19 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - stage: ConsolidateOdbc - displayName: 'Consolidate All ODBC Artifacts' - dependsOn: - - ODBC_Win_x64 - - ODBC_Win_arm64 - - ODBC_MacOS_universal2 - - ODBC_Linux_manylinux_2_28_x86_64 - - ODBC_Linux_manylinux_2_28_aarch64 - - ODBC_Linux_musllinux_x86_64 - - ODBC_Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self - parameters: - oneBranchType: '${{ variables.effectiveOneBranchType }}' - expectedWheelCount: 7 + - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 From 72c3c884eae523fa79d71aea7ab24883113bbfba Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 10 Jul 2026 15:21:13 +0530 Subject: [PATCH 26/35] build: nightly build always builds both packages Add effectiveBuildPackage variable that forces 'both' on scheduled (Build.Reason=Schedule) runs while manual/PR runs honour the buildPackage parameter. Mirrors the effectiveOneBranchType pattern so the nightly keeps refreshing every artifact. --- .../build-release-package-pipeline.yml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 29e735080..c485cc8ee 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -76,7 +76,8 @@ parameters: # mssql-python -> Windows/macOS/Linux wheels + Consolidate # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc # both -> everything (previous default behaviour) - # Scheduled (daily) builds use this default too, so they build mssql-python only. + # Scheduled (daily) builds ignore this and always build 'both' (see + # effectiveBuildPackage below) so the nightly keeps producing every artifact. - name: buildPackage displayName: 'Package to Build' type: string @@ -192,6 +193,14 @@ variables: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' + + # Determine which package(s) to build: scheduled (nightly) builds always build + # 'both' so every artifact keeps refreshing; manual/PR builds honour the parameter. + - name: effectiveBuildPackage + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'both' + ${{ else }}: + value: '${{ parameters.buildPackage }}' # Variable template imports # Each file provides specific variable groups: @@ -401,7 +410,7 @@ extends: # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning # Gate: build mssql-python stages unless an odbc-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.windowsConfigs }}: - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self parameters: @@ -428,7 +437,7 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.macosConfigs }}: - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self parameters: @@ -459,7 +468,7 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - ${{ each config in parameters.linuxConfigs }}: - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self parameters: @@ -482,7 +491,7 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - ${{ if ne(parameters.buildPackage, 'mssql-python-odbc') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: - stage: Consolidate displayName: 'Consolidate All Artifacts' dependsOn: @@ -529,7 +538,7 @@ extends: # # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcWindowsConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self parameters: @@ -539,7 +548,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC macOS stage (1): ODBC_MacOS_universal2 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcMacosConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self parameters: @@ -548,7 +557,7 @@ extends: oneBranchType: '${{ variables.effectiveOneBranchType }}' # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - ${{ each config in parameters.odbcLinuxConfigs }}: - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self parameters: @@ -566,7 +575,7 @@ extends: # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release # pipeline selects one artifact based on its `releasePackage` parameter. - - ${{ if ne(parameters.buildPackage, 'mssql-python') }}: + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: - stage: ConsolidateOdbc displayName: 'Consolidate All ODBC Artifacts' dependsOn: From b15c8ad2ff7656d8e9c0a4b7528b13e85c94f49c Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 17 Jul 2026 16:58:48 +0530 Subject: [PATCH 27/35] CHORE: reconcile GH pipeline Option A both-wheels test with validated ADO design Add hardened wheel-installation-test-stage.yml (positive+negative driver proof, source-tree shadowing guards, TESTED==0 fails stage) and wire two 'both'-gated TestBothWheels_* stages (manylinux_2_28 + musllinux x86_64) after ConsolidateOdbc, replacing the old single 3.12 invocation. --- .../build-release-package-pipeline.yml | 27 ++ .../stages/wheel-installation-test-stage.yml | 316 ++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 OneBranchPipelines/stages/wheel-installation-test-stage.yml diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index c485cc8ee..9c505f009 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -591,3 +591,30 @@ extends: parameters: oneBranchType: '${{ variables.effectiveOneBranchType }}' expectedWheelCount: 7 + + # ========================================================================= + # WHEEL INSTALLATION TEST STAGE(S) + # ========================================================================= + # Post-consolidation validation of the package split from an end user's + # perspective: install the ACTUAL published mssql-python + mssql-python-odbc + # wheels together, remove the bundled ODBC driver so the external package is + # the ONLY driver source, then run the full pytest suite. Runs only when BOTH + # packages are built in the same run (buildPackage = 'both'), because it needs + # both consolidated artifacts (drop_Consolidate_* and drop_ConsolidateOdbc_*). + - ${{ if eq(variables.effectiveBuildPackage, 'both') }}: + - template: /OneBranchPipelines/stages/wheel-installation-test-stage.yml@self + parameters: + stageName: TestBothWheels_manylinux_2_28_x86_64 + jobName: InstallAndTest + linuxTag: manylinux_2_28 + arch: x86_64 + dockerPlatform: linux/amd64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' + - template: /OneBranchPipelines/stages/wheel-installation-test-stage.yml@self + parameters: + stageName: TestBothWheels_musllinux_x86_64 + jobName: InstallAndTest + linuxTag: musllinux + arch: x86_64 + dockerPlatform: linux/amd64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' diff --git a/OneBranchPipelines/stages/wheel-installation-test-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-stage.yml new file mode 100644 index 000000000..7534f67f7 --- /dev/null +++ b/OneBranchPipelines/stages/wheel-installation-test-stage.yml @@ -0,0 +1,316 @@ +# ========================================================================================= +# End-User "Both Wheels Together" Installation Test Stage +# ========================================================================================= +# Validates the mssql-python / mssql-python-odbc PACKAGE SPLIT from an end-user's +# perspective, using the ACTUAL published wheel artifacts (not raw build binaries): +# +# pip install mssql_python---.whl +# pip install mssql_python_odbc--py3-none-.whl +# +# The proof is both POSITIVE and NEGATIVE: +# * POSITIVE: before touching anything we assert that the native loader WILL pick +# the external package - GetDriverPathCpp(dir(mssql_python_odbc)) points at an +# existing driver file, which is exactly the condition GetOdbcLibsBaseDir uses +# to treat the external package as authoritative. +# * NEGATIVE: we then DELETE the ODBC driver binaries that mssql-python still +# bundles under `mssql_python/libs/`. With the bundled fallback removed, a +# successful connection + passing test suite proves the driver was resolved +# from the separately-installed `mssql_python_odbc` package. +# (see GetOdbcLibsBaseDir in ddbc_bindings.cpp, which prefers the external package +# and falls back to bundled libs). Without the deletion the bundled fallback would +# mask a broken split and the test would pass even if the ODBC package were ignored. +# +# Scope: Linux manylinux_2_28 + musllinux (x86_64), all supported Python versions. +# This stage tests the INSTALL/RESOLUTION path, not per-arch compilation (already +# covered by the build stages). It runs only when BOTH packages are built in the +# same run (buildPackage = 'both'), because it needs both consolidated artifacts. +# ========================================================================================= +parameters: + # Stage identifier (e.g., 'TestBothWheels_manylinux_2_28_x86_64') + - name: stageName + type: string + # Job identifier within the stage + - name: jobName + type: string + default: 'InstallAndTest' + # Linux distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc) + - name: linuxTag + type: string + # CPU architecture (test stage targets x86_64 only) + - name: arch + type: string + default: 'x86_64' + # Docker platform for the build/test container + - name: dockerPlatform + type: string + default: 'linux/amd64' + # OneBranch build type: 'Official' or 'NonOfficial' + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Test Both Wheels ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Only meaningful when both consolidated artifacts exist in this run. + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Install both wheels + pytest - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 90 + + variables: + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + # Download the two CONSOLIDATED wheel artifacts produced earlier in this run: + # drop_Consolidate_ConsolidateArtifacts -> mssql-python wheels (all platforms/pyvers) + # drop_ConsolidateOdbc_ConsolidateArtifacts -> mssql-python-odbc wheels (7 platforms) + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python wheels' + inputs: + buildType: 'current' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-wheels' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python-odbc wheels' + inputs: + buildType: 'current' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-odbc-wheels' + + - bash: | + set -euxo pipefail + if ! docker info > /dev/null 2>&1; then + echo "Docker daemon not running, starting it..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Ensure Docker daemon is running' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'" >&2 + exit 1 + fi + + # Mount the repo and downloaded wheels read-only. Read-only /workspace is + # deliberate: this stage must exercise the INSTALLED wheels, never the + # source checkout, so nothing here should ever write into the repo tree. + docker run -d --name test-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace:ro \ + -v $(Pipeline.Workspace)/mssql-python-wheels:/wheels/mssql-python:ro \ + -v $(Pipeline.Workspace)/mssql-python-odbc-wheels:/wheels/mssql-python-odbc:ro \ + -w /workspace \ + $IMAGE \ + tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) test container' + + # Start a SQL Server 2022 container on the host so pytest has a live server. + - script: | + set -euxo pipefail + docker run -d --name sqlserver-$(LINUX_TAG)-$(ARCH) \ + --platform linux/amd64 \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ + -p 1433:1433 \ + mcr.microsoft.com/mssql/server:2022-latest + + echo "Waiting for SQL Server to be ready..." + for i in {1..30}; do + if docker exec sqlserver-$(LINUX_TAG)-$(ARCH) /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server is ready" + break + fi + sleep 2 + done + + SQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sqlserver-$(LINUX_TAG)-$(ARCH)) + echo "SQL Server IP: $SQL_IP" + echo "##vso[task.setvariable variable=SQL_IP]$SQL_IP" + displayName: 'Start SQL Server container for testing' + env: + DB_PASSWORD: $(DB_PASSWORD) + + # For each Python version: install BOTH wheels, remove the bundled ODBC + # driver from the installed mssql_python package, then run pytest. A pass + # proves the driver was loaded from the separate mssql_python_odbc package. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + + # Count how many Python versions actually ran the full test so the stage + # cannot go green having silently skipped every version (e.g. if none of + # the /opt/python/ interpreters exist in the image). + TESTED=0 + # Inner script uses exit code 3 to signal "interpreter absent - skipped". + SKIP_RC=3 + + for PYBIN in cp310 cp311 cp312 cp313 cp314; do + echo "" + echo "=====================================================" + echo "End-user both-wheels test: $PYBIN on $(LINUX_TAG)/$(ARCH)" + echo "=====================================================" + + set +e + docker exec -e PYBIN=$PYBIN -e LINUX_TAG="$(LINUX_TAG)" -e SQL_IP=$(SQL_IP) -e DB_PASSWORD="$(DB_PASSWORD)" \ + test-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -euo pipefail + + PY=/opt/python/${PYBIN}-${PYBIN}/bin/python + test -x "$PY" || { echo "Python $PY missing - skipping ${PYBIN}"; exit 3; } + echo "Using: $($PY --version)" + + # Fresh isolated venv = clean end-user machine + TEST_DIR="/enduser_${PYBIN}" + rm -rf "$TEST_DIR" + $PY -m venv "$TEST_DIR" + VENV_PY="$TEST_DIR/bin/python" + + # Run all Python invocations from the venv dir (never /workspace) and + # with PYTHONSAFEPATH so the repo checkout at /workspace/mssql_python + # can NEVER shadow the freshly installed wheel. Without this, cwd + # (/workspace) lands on sys.path and "import mssql_python" would resolve + # to the source tree instead of the package under test. + export PYTHONSAFEPATH=1 + cd "$TEST_DIR" + + $VENV_PY -m pip install -q -U pip + + # Platform wheel tag: manylinux_2_28 vs musllinux_1_2, x86_64 + if [ "$LINUX_TAG" = "manylinux_2_28" ]; then PLAT="manylinux"; else PLAT="musllinux"; fi + + # Select the mssql-python wheel matching THIS Python version + libc + MP_WHEEL=$(ls /wheels/mssql-python/dist/mssql_python-*${PYBIN}-*${PLAT}*x86_64*.whl 2>/dev/null | head -1) + if [ -z "$MP_WHEEL" ]; then + echo "ERROR: no mssql-python wheel for ${PYBIN}/${PLAT}"; exit 1 + fi + + # ODBC wheel is data-only (py3-none-), one per platform + ODBC_WHEEL=$(ls /wheels/mssql-python-odbc/dist/mssql_python_odbc-*${PLAT}*x86_64*.whl 2>/dev/null | head -1) + if [ -z "$ODBC_WHEEL" ]; then + echo "ERROR: no mssql-python-odbc wheel for ${PLAT}"; exit 1 + fi + + echo "Installing mssql-python: $MP_WHEEL" + $VENV_PY -m pip install -q "$MP_WHEEL" + echo "Installing mssql-python-odbc: $ODBC_WHEEL" + $VENV_PY -m pip install -q "$ODBC_WHEEL" + + # Show where the external ODBC package lives (the split target) + $VENV_PY -c "import mssql_python_odbc, os; print(\"mssql_python_odbc libs:\", mssql_python_odbc.get_libs_dir()); assert os.path.isdir(mssql_python_odbc.get_libs_dir())" + + # POSITIVE assertion: prove the native loader WILL resolve the driver + # from the external mssql_python_odbc package. This mirrors exactly what + # GetOdbcLibsBaseDir() does in ddbc_bindings.cpp: it takes the parent dir + # of mssql_python_odbc.__file__ as the base and treats the external + # package as authoritative iff GetDriverPathCpp(base) exists. If this + # driver file is present, the loader is guaranteed to pick the external + # package (non-Windows only checks the driver; Windows also needs + # mssql-auth.dll, which this platform test does not cover). + $VENV_PY -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print(\"external driver path:\", drv); assert os.path.exists(drv), \"external ODBC driver not found - loader would NOT select external package: \"+drv; print(\"POSITIVE: loader will resolve the driver from the external mssql_python_odbc package\")" + + # DECISIVE (negative) STEP: remove the ODBC driver bundled inside + # mssql_python. With the fallback gone, any successful connection must + # be using the driver from the separate mssql_python_odbc package. + MP_LIBS=$($VENV_PY -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), \"libs\"))") + case "$MP_LIBS" in + /workspace/*) + echo "ERROR: resolved mssql_python to the source tree ($MP_LIBS), not the installed wheel"; exit 1 ;; + esac + if [ -d "$MP_LIBS" ]; then + echo "Removing bundled mssql_python/libs to force use of external ODBC package: $MP_LIBS" + rm -rf "$MP_LIBS" + else + echo "Note: mssql_python has no bundled libs/ (already split at build time)" + fi + + # Smoke test: import + live connection using ONLY the external driver + $VENV_PY -c "import mssql_python; print(\"mssql_python\", mssql_python.__version__, \"imported OK\")" + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -c "import mssql_python, os; c=mssql_python.connect(os.environ[\"DB_CONNECTION_STRING\"]); cur=c.cursor(); cur.execute(\"SELECT 1\"); print(\"connection OK:\", cur.fetchone()); c.close()" + + # Copy the suite OUT of /workspace so pytest collection cannot import + # the source-tree package either, then run against the installed wheels. + cp -r /workspace/tests "$TEST_DIR/tests" + cp /workspace/pytest.ini "$TEST_DIR/pytest.ini" 2>/dev/null || true + $VENV_PY -m pip install -q pytest + if [ -f /workspace/requirements.txt ]; then + $VENV_PY -m pip install -q -r /workspace/requirements.txt + fi + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -m pytest "$TEST_DIR/tests" -v --maxfail=1 + echo "All tests passed for ${PYBIN} using the EXTERNAL mssql_python_odbc driver" + ' + rc=$? + set -e + + if [ "$rc" -eq "$SKIP_RC" ]; then + echo "Skipped $PYBIN (interpreter not present in image)" + elif [ "$rc" -ne 0 ]; then + echo "ERROR: both-wheels test failed for $PYBIN (exit $rc)" + exit "$rc" + else + TESTED=$((TESTED + 1)) + echo "Both-wheels test complete for $PYBIN" + fi + done + + if [ "$TESTED" -eq 0 ]; then + echo "ERROR: no Python versions were tested - every interpreter was missing. Failing the stage." + exit 1 + fi + + echo "=====================================================" + echo "All available Python versions ($TESTED) passed the both-wheels end-user test" + echo "=====================================================" + displayName: 'Install both wheels + pytest (Python 3.10-3.14)' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + docker stop test-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + docker rm test-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Cleanup containers' + condition: always() From 7795a2d94fb477adc70f98b9570156f99c02df55 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Mon, 20 Jul 2026 14:13:11 +0530 Subject: [PATCH 28/35] FIX: resolve DevSkim localhost finding and harden both-wheels test stage - Suppress DevSkim DS162092 inline on the in-container sqlcmd readiness probe (localhost is the mssql container's own loopback, not debug code). - Reorder to run the full suite with bundled libs intact (external ODBC driver is already preferred) then a decisive post-delete SELECT 1 connect. - Make pytest.ini copy mandatory and add explicit -m 'not stress'. --- .../stages/wheel-installation-test-stage.yml | 84 ++++++++++++------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/OneBranchPipelines/stages/wheel-installation-test-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-stage.yml index 7534f67f7..e19e951e3 100644 --- a/OneBranchPipelines/stages/wheel-installation-test-stage.yml +++ b/OneBranchPipelines/stages/wheel-installation-test-stage.yml @@ -11,19 +11,25 @@ # * POSITIVE: before touching anything we assert that the native loader WILL pick # the external package - GetDriverPathCpp(dir(mssql_python_odbc)) points at an # existing driver file, which is exactly the condition GetOdbcLibsBaseDir uses -# to treat the external package as authoritative. -# * NEGATIVE: we then DELETE the ODBC driver binaries that mssql-python still -# bundles under `mssql_python/libs/`. With the bundled fallback removed, a -# successful connection + passing test suite proves the driver was resolved -# from the separately-installed `mssql_python_odbc` package. -# (see GetOdbcLibsBaseDir in ddbc_bindings.cpp, which prefers the external package -# and falls back to bundled libs). Without the deletion the bundled fallback would -# mask a broken split and the test would pass even if the ODBC package were ignored. +# to treat the external package as authoritative. Because that resolver prefers +# the external package on EVERY connect whenever its driver is complete - +# regardless of whether mssql_python still bundles libs (see GetOdbcLibsBaseDir +# in ddbc_bindings.cpp) - simply installing the odbc wheel already exercises the +# external-resolution branch throughout the whole suite. +# * NEGATIVE: AFTER the full suite passes, we DELETE the ODBC driver binaries that +# mssql-python still bundles under `mssql_python/libs/` and connect once more. +# With the bundled fallback removed, a successful live query can ONLY be using +# the driver from the separately-installed `mssql_python_odbc` package. We do +# NOT re-run the full suite after deletion: test_000_dependencies intentionally +# asserts the bundled package layout, which this negative step deliberately breaks. # # Scope: Linux manylinux_2_28 + musllinux (x86_64), all supported Python versions. # This stage tests the INSTALL/RESOLUTION path, not per-arch compilation (already # covered by the build stages). It runs only when BOTH packages are built in the # same run (buildPackage = 'both'), because it needs both consolidated artifacts. +# NOTE: this external-resolution coverage is nightly-only. The PR-validation +# pipeline never installs the odbc wheel, so its connects still use the bundled +# driver path - that gap is intentional and covered here on the daily build. # ========================================================================================= parameters: # Stage identifier (e.g., 'TestBothWheels_manylinux_2_28_x86_64') @@ -66,7 +72,9 @@ stages: name: Python-1ES-pool demands: - imageOverride -equals PYTHON-1ES-UB2404 - timeoutInMinutes: 90 + # 5 Python versions x full pytest suite (+ venv/pip/install overhead) per + # stage; headroom over the observed runtime so a slow agent cannot time out. + timeoutInMinutes: 150 variables: - name: ob_sdl_binskim_enabled @@ -158,8 +166,10 @@ stages: echo "Waiting for SQL Server to be ready..." for i in {1..30}; do + # sqlcmd runs INSIDE the mssql container via `docker exec`; localhost is the + # container's own loopback (a readiness probe), not debug code reaching a dev box. if docker exec sqlserver-$(LINUX_TAG)-$(ARCH) /opt/mssql-tools18/bin/sqlcmd \ - -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then + -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then # DevSkim: ignore DS162092 echo "SQL Server is ready" break fi @@ -173,8 +183,9 @@ stages: env: DB_PASSWORD: $(DB_PASSWORD) - # For each Python version: install BOTH wheels, remove the bundled ODBC - # driver from the installed mssql_python package, then run pytest. A pass + # For each Python version: install BOTH wheels, run the full pytest suite + # with bundled libs intact (external driver already preferred), THEN remove + # the bundled ODBC driver and connect once more. The post-deletion connect # proves the driver was loaded from the separate mssql_python_odbc package. - script: | set -euxo pipefail @@ -251,9 +262,34 @@ stages: # mssql-auth.dll, which this platform test does not cover). $VENV_PY -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print(\"external driver path:\", drv); assert os.path.exists(drv), \"external ODBC driver not found - loader would NOT select external package: \"+drv; print(\"POSITIVE: loader will resolve the driver from the external mssql_python_odbc package\")" - # DECISIVE (negative) STEP: remove the ODBC driver bundled inside - # mssql_python. With the fallback gone, any successful connection must - # be using the driver from the separate mssql_python_odbc package. + # PHASE 1: run the FULL existing suite with bundled libs INTACT. + # The external mssql_python_odbc package is already installed, so + # GetOdbcLibsBaseDir() prefers it on every connect (it selects the + # external package whenever its driver is complete, regardless of + # whether mssql_python still bundles libs - see ddbc_bindings.cpp). + # This exercises the external-resolution branch throughout while + # keeping tests that assert the bundled package layout (e.g. + # test_000_dependencies) valid. Copy the suite OUT of /workspace so + # pytest collection cannot import the source-tree package. + cp -r /workspace/tests "$TEST_DIR/tests" + # pytest.ini carries the `-m "not stress"` default and markers; it must + # be present so collection behaves like a normal repo run. Fail loudly + # if it is missing rather than silently running an unconfigured suite. + cp /workspace/pytest.ini "$TEST_DIR/pytest.ini" + $VENV_PY -m pip install -q pytest + if [ -f /workspace/requirements.txt ]; then + $VENV_PY -m pip install -q -r /workspace/requirements.txt + fi + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -m pytest "$TEST_DIR/tests" -v --maxfail=1 -m "not stress" + echo "Full suite passed for ${PYBIN} with both wheels installed (external driver preferred)" + + # PHASE 2 - DECISIVE negative proof: remove the ODBC driver that + # mssql_python still bundles, then connect again. With the bundled + # fallback gone, a successful live query can ONLY be using the driver + # from the separate mssql_python_odbc package. We do NOT re-run the + # full suite here: test_000_dependencies intentionally asserts the + # bundled layout, which this step deliberately breaks. MP_LIBS=$($VENV_PY -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), \"libs\"))") case "$MP_LIBS" in /workspace/*) @@ -266,22 +302,10 @@ stages: echo "Note: mssql_python has no bundled libs/ (already split at build time)" fi - # Smoke test: import + live connection using ONLY the external driver - $VENV_PY -c "import mssql_python; print(\"mssql_python\", mssql_python.__version__, \"imported OK\")" - DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ - $VENV_PY -c "import mssql_python, os; c=mssql_python.connect(os.environ[\"DB_CONNECTION_STRING\"]); cur=c.cursor(); cur.execute(\"SELECT 1\"); print(\"connection OK:\", cur.fetchone()); c.close()" - - # Copy the suite OUT of /workspace so pytest collection cannot import - # the source-tree package either, then run against the installed wheels. - cp -r /workspace/tests "$TEST_DIR/tests" - cp /workspace/pytest.ini "$TEST_DIR/pytest.ini" 2>/dev/null || true - $VENV_PY -m pip install -q pytest - if [ -f /workspace/requirements.txt ]; then - $VENV_PY -m pip install -q -r /workspace/requirements.txt - fi + $VENV_PY -c "import mssql_python; print(\"mssql_python\", mssql_python.__version__, \"imported OK (bundled libs removed)\")" DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ - $VENV_PY -m pytest "$TEST_DIR/tests" -v --maxfail=1 - echo "All tests passed for ${PYBIN} using the EXTERNAL mssql_python_odbc driver" + $VENV_PY -c "import mssql_python, os; c=mssql_python.connect(os.environ[\"DB_CONNECTION_STRING\"]); cur=c.cursor(); cur.execute(\"SELECT 1\"); print(\"connection OK via EXTERNAL driver:\", cur.fetchone()); c.close()" + echo "DECISIVE: ${PYBIN} connected with bundled libs removed - driver resolved from external mssql_python_odbc package" ' rc=$? set -e From 6f60f306d28741395d69264ea758504fead7ce72 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 21 Jul 2026 18:02:38 +0530 Subject: [PATCH 29/35] FIX: add Windows + macOS both-wheels install tests and krb5 deps for musllinux - Add wheel-installation-test-windows-stage.yml (native Windows both-wheels install/test on LocalDB) - Add wheel-installation-test-macos-stage.yml (macOS-15 both-wheels install/test; wheels retagged macosx_15_0) - Install krb5-libs/keyutils-libs in Linux both-wheels test container (msodbcsql18 runtime dep; fixes musllinux core dump) - Wire both new stages into build-release-package-pipeline.yml under the 'both' gate --- .../build-release-package-pipeline.yml | 27 ++- .../wheel-installation-test-macos-stage.yml | 207 ++++++++++++++++++ .../stages/wheel-installation-test-stage.yml | 30 ++- .../wheel-installation-test-windows-stage.yml | 196 +++++++++++++++++ 4 files changed, 457 insertions(+), 3 deletions(-) create mode 100644 OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml create mode 100644 OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 4a790b70d..2c39429ef 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -378,7 +378,13 @@ extends: # Version automatically detected from wheel metadata (setup.py) sbom: enabled: ${{ parameters.runSdlTasks }} - packageName: 'mssql-python' + # Name the SBOM after the package actually being built so an odbc-only build + # does not mislabel the driver wheels as 'mssql-python'. Combined ('both') and + # mssql-python builds keep the primary package name. + ${{ if eq(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + packageName: 'mssql-python-odbc' + ${{ else }}: + packageName: 'mssql-python' # TSA - Threat and Security Assessment # Uploads scan results to Microsoft's TSA tool for tracking @@ -618,3 +624,22 @@ extends: arch: x86_64 dockerPlatform: linux/amd64 oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Windows + macOS counterparts run natively (no Docker) on the same pools the + # build stages use. They prove the external-package resolution end-to-end on all + # three OSes - Windows additionally covers mssql-auth.dll resolution, macOS covers + # the universal2 dylib @loader_path/rpath resolution. + - template: /OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml@self + parameters: + stageName: TestBothWheels_Windows_x64 + jobName: InstallAndTest + pythonVersion: '3.12' + shortPyVer: '312' + architecture: x64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' + - template: /OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml@self + parameters: + stageName: TestBothWheels_macOS_universal2 + jobName: InstallAndTest + pythonVersion: '3.12' + shortPyVer: '312' + oneBranchType: '${{ variables.effectiveOneBranchType }}' diff --git a/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml new file mode 100644 index 000000000..5b923f0fa --- /dev/null +++ b/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml @@ -0,0 +1,207 @@ +# ========================================================================================= +# End-User "Both Wheels Together" Installation Test - macOS +# ========================================================================================= +# macOS counterpart of wheel-installation-test-stage.yml (which is Docker/Linux-only). +# Installs the ACTUAL published mssql-python (universal2) + mssql-python-odbc (universal2) +# wheels into a fresh venv, runs the full pytest suite (bundled libs intact, external +# package already preferred by GetOdbcLibsBaseDir), then DELETES the bundled +# mssql_python/libs and connects once more in a fresh process. With the bundled fallback +# gone, a successful live query can ONLY be using the driver from the separately installed +# mssql_python_odbc package. This also exercises the universal2 dylib @loader_path/rpath +# resolution from the external package. +# +# Runs natively on a macOS-15 + Colima/Docker SQL Server agent. macOS 15 is required +# because the universal2 wheels are retagged to a macosx_15_0 minimum deployment target. +# A single representative Python version is sufficient: the external package RESOLUTION +# path (GetOdbcLibsBaseDir in ddbc_bindings.cpp) is Python-version independent; per-version +# native builds are already covered by the macOS build stages. +# ========================================================================================= +parameters: + # Stage identifier (e.g., 'TestBothWheels_macOS_universal2') + - name: stageName + type: string + # Job identifier within the stage + - name: jobName + type: string + default: 'InstallAndTest' + # Python version in X.Y format used to select the mssql-python wheel + - name: pythonVersion + type: string + default: '3.12' + # Python version as 3-digit string (e.g., '312') matching the cp wheel tag + - name: shortPyVer + type: string + default: '312' + # OneBranch build type: 'Official' or 'NonOfficial' + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Test Both Wheels macOS universal2' + # Only meaningful when both consolidated artifacts exist in this run. + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Install both wheels + pytest - macOS universal2' + + # macOS-15 (Sequoia). type:linux is an Azure Pipelines quirk - macOS pools + # declare as 'linux' type. The build stage retags the universal2 wheels to + # macosx_15_0 (min deployment target), so pip only accepts them on macOS 15+. + # The build stage itself runs on macOS-14 because it tests the freshly built + # .so directly (never pip-installs the retagged wheel); this install test does + # pip-install it, so it MUST run on macOS 15+ or pip rejects the wheel with + # "not a supported wheel on this platform". + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-15' + timeoutInMinutes: 120 + + variables: + # BinSkim is Windows-PE focused; macOS uses Mach-O. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS agents). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + # Download the two CONSOLIDATED wheel artifacts produced earlier in this run. + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python wheels' + inputs: + buildType: 'current' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-wheels' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python-odbc wheels' + inputs: + buildType: 'current' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-odbc-wheels' + + # Install Docker CLI + Colima (macOS Docker runtime) and start a SQL Server + # 2022 container on the host loopback (localhost:1433), same as the build stage. + - script: | + brew update + brew install docker colima + colima start --vm-type vz --cpu 4 --memory 8 || { + echo "vz VM failed, trying qemu..." + colima start --vm-type qemu --cpu 4 --memory 8 + } + sleep 30 + docker context use colima >/dev/null || true + docker version + displayName: 'Install and start Docker (Colima)' + timeoutInMinutes: 15 + + - script: | + set -euxo pipefail + docker pull mcr.microsoft.com/mssql/server:2022-latest + docker run --name sqlserver \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \ + -p 1433:1433 -d \ + mcr.microsoft.com/mssql/server:2022-latest + echo "Waiting for SQL Server to be ready..." + for i in {1..30}; do + # sqlcmd runs INSIDE the mssql container; localhost is the container's own + # loopback (a readiness probe), not debug code reaching a dev box. + if docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$DB_PASSWORD" -C -Q "SELECT 1" >/dev/null 2>&1; then # DevSkim: ignore DS162092 + echo "SQL Server is ready"; break + fi + sleep 2 + done + displayName: 'Start SQL Server (Docker)' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + set -euo pipefail + + PYTAG="cp${{ parameters.shortPyVer }}" + + # Select the universal2 wheels matching THIS Python version (mssql-python) + # and the data-only odbc package (py3-none-macosx-universal2). + MP_WHEEL=$(ls "$(Pipeline.Workspace)"/mssql-python-wheels/dist/mssql_python-*${PYTAG}-*macosx*universal2*.whl 2>/dev/null | head -1) + if [ -z "$MP_WHEEL" ]; then echo "ERROR: no mssql-python wheel for ${PYTAG}/macosx universal2"; exit 1; fi + ODBC_WHEEL=$(ls "$(Pipeline.Workspace)"/mssql-python-odbc-wheels/dist/mssql_python_odbc-*macosx*universal2*.whl 2>/dev/null | head -1) + if [ -z "$ODBC_WHEEL" ]; then echo "ERROR: no mssql-python-odbc wheel for macosx universal2"; exit 1; fi + + # Fresh isolated venv = clean end-user machine. + TEST_DIR="$(Agent.TempDirectory)/enduser_${PYTAG}" + rm -rf "$TEST_DIR" + python3 -m venv "$TEST_DIR" + VENV_PY="$TEST_DIR/bin/python" + + # PYTHONSAFEPATH so the repo checkout can never shadow the installed wheel. + export PYTHONSAFEPATH=1 + + "$VENV_PY" -m pip install -q -U pip + + # Install the odbc wheel FIRST so mssql-python's pinned mssql-python-odbc==X + # requirement is satisfied from this LOCAL wheel (pip never reaches PyPI). + echo "Installing mssql-python-odbc: $ODBC_WHEEL" + "$VENV_PY" -m pip install -q "$ODBC_WHEEL" + echo "Installing mssql-python: $MP_WHEEL" + "$VENV_PY" -m pip install -q "$MP_WHEEL" + + # Show where the external ODBC package lives (the split target). + "$VENV_PY" -c "import mssql_python_odbc, os; print('mssql_python_odbc libs:', mssql_python_odbc.get_libs_dir()); assert os.path.isdir(mssql_python_odbc.get_libs_dir())" + + # POSITIVE assertion: prove the native loader WILL resolve the driver from + # the external mssql_python_odbc package (mirrors GetOdbcLibsBaseDir). + "$VENV_PY" -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print('external driver path:', drv); assert os.path.exists(drv), 'external ODBC driver not found - loader would NOT select external package: '+drv; print('POSITIVE: loader will resolve the driver from the external mssql_python_odbc package')" + + # PHASE 1: run the FULL existing suite with bundled libs INTACT. Copy the + # suite OUT of the source tree so collection imports the INSTALLED package. + cp -r "$(Build.SourcesDirectory)/tests" "$TEST_DIR/tests" + cp "$(Build.SourcesDirectory)/pytest.ini" "$TEST_DIR/pytest.ini" + "$VENV_PY" -m pip install -q pytest + if [ -f "$(Build.SourcesDirectory)/requirements.txt" ]; then + "$VENV_PY" -m pip install -q -r "$(Build.SourcesDirectory)/requirements.txt" + fi + cd "$TEST_DIR" + DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + "$VENV_PY" -m pytest "$TEST_DIR/tests" -v --maxfail=1 -m "not stress" + echo "Full suite passed for ${PYTAG} with both wheels installed (external driver preferred)" + + # PHASE 2 - DECISIVE negative proof: remove the ODBC driver that + # mssql_python still bundles, then connect again in a FRESH process. + MP_LIBS=$("$VENV_PY" -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), 'libs'))") + case "$MP_LIBS" in + "$(Build.SourcesDirectory)"/*) + echo "ERROR: resolved mssql_python to the source tree ($MP_LIBS), not the installed wheel"; exit 1 ;; + esac + if [ -d "$MP_LIBS" ]; then + echo "Removing bundled mssql_python/libs to force use of external ODBC package: $MP_LIBS" + rm -rf "$MP_LIBS" + else + echo "Note: mssql_python has no bundled libs/ (already split at build time)" + fi + + "$VENV_PY" -c "import mssql_python; print('mssql_python', mssql_python.__version__, 'imported OK (bundled libs removed)')" + DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + "$VENV_PY" -c "import mssql_python, os; c=mssql_python.connect(os.environ['DB_CONNECTION_STRING']); cur=c.cursor(); cur.execute('SELECT 1'); print('connection OK via EXTERNAL driver:', cur.fetchone()); c.close()" + echo "DECISIVE: connected with bundled libs removed - driver resolved from external mssql_python_odbc package" + displayName: 'Install both wheels + full pytest + external-only proof' + env: + DB_PASSWORD: $(DB_PASSWORD) diff --git a/OneBranchPipelines/stages/wheel-installation-test-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-stage.yml index e19e951e3..3513f4ed1 100644 --- a/OneBranchPipelines/stages/wheel-installation-test-stage.yml +++ b/OneBranchPipelines/stages/wheel-installation-test-stage.yml @@ -154,6 +154,28 @@ stages: tail -f /dev/null displayName: 'Start $(LINUX_TAG) $(ARCH) test container' + # Install the ODBC driver's runtime SYSTEM dependencies INTO the test + # container. msodbcsql18 links MIT Kerberos (libkrb5.so.3 / + # libgssapi_krb5.so.2) as a system dependency - these are never bundled in + # the wheel (the driver's RUNPATH is only $ORIGIN, which resolves the + # co-located libodbcinst.so.2 but NOT krb5). The manylinux_2_28 base image + # ships krb5-libs by default, but the minimal musllinux/Alpine image does + # NOT, so without this the driver's dlopen fails with + # "Error loading shared library libkrb5.so.3: No such file or directory" and, + # because that throw happens inside std::call_once on musl, the process + # aborts (core dump) at import instead of raising a clean ImportError. This + # mirrors the krb5-libs install already performed by + # build-linux-single-stage.yml and reflects the real runtime requirement for + # Alpine end users. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "musllinux" ]]; then + docker exec test-$(LINUX_TAG)-$(ARCH) sh -lc 'apk add --no-cache krb5-libs keyutils-libs' + else + docker exec test-$(LINUX_TAG)-$(ARCH) bash -lc 'dnf -y install krb5-libs keyutils-libs || yum -y install krb5-libs keyutils-libs || true' + fi + displayName: 'Install ODBC driver runtime deps (krb5) in $(LINUX_TAG) container' + # Start a SQL Server 2022 container on the host so pytest has a live server. - script: | set -euxo pipefail @@ -244,10 +266,14 @@ stages: echo "ERROR: no mssql-python-odbc wheel for ${PLAT}"; exit 1 fi - echo "Installing mssql-python: $MP_WHEEL" - $VENV_PY -m pip install -q "$MP_WHEEL" + # Install the odbc wheel FIRST so that once mssql-python pins + # mssql-python-odbc==X (setup.py install_requires), the requirement + # is already satisfied from this LOCAL wheel and pip never reaches + # PyPI for a version that may not be published yet. echo "Installing mssql-python-odbc: $ODBC_WHEEL" $VENV_PY -m pip install -q "$ODBC_WHEEL" + echo "Installing mssql-python: $MP_WHEEL" + $VENV_PY -m pip install -q "$MP_WHEEL" # Show where the external ODBC package lives (the split target) $VENV_PY -c "import mssql_python_odbc, os; print(\"mssql_python_odbc libs:\", mssql_python_odbc.get_libs_dir()); assert os.path.isdir(mssql_python_odbc.get_libs_dir())" diff --git a/OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml new file mode 100644 index 000000000..8a45e39b7 --- /dev/null +++ b/OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml @@ -0,0 +1,196 @@ +# ========================================================================================= +# End-User "Both Wheels Together" Installation Test - WINDOWS +# ========================================================================================= +# Windows counterpart of wheel-installation-test-stage.yml (which is Docker/Linux-only). +# Installs the ACTUAL published mssql-python + mssql-python-odbc Windows wheels into a +# fresh venv, runs the full pytest suite (bundled libs intact, external package already +# preferred by GetOdbcLibsBaseDir), then DELETES the bundled mssql_python/libs and +# connects once more in a fresh process. With the bundled fallback gone, a successful +# live query can ONLY be using the driver from the separately installed +# mssql_python_odbc package. +# +# Windows adds coverage the Linux stage explicitly cannot: on Windows the loader treats +# the external package as authoritative only when BOTH the ODBC driver AND mssql-auth.dll +# are present, so this proves the auth DLL is resolved from the external package too. +# +# Runs natively on the same 1ES Windows Server + SQL Server LocalDB pool as the build +# stage (no Docker). A single representative Python version is sufficient: the external +# package RESOLUTION path (GetOdbcLibsBaseDir in ddbc_bindings.cpp) is Python-version +# independent; per-version native builds are already covered by the Windows build stages. +# ========================================================================================= +parameters: + # Stage identifier (e.g., 'TestBothWheels_Windows_x64') + - name: stageName + type: string + # Job identifier within the stage + - name: jobName + type: string + default: 'InstallAndTest' + # Python version in X.Y format used to select the mssql-python wheel + - name: pythonVersion + type: string + default: '3.12' + # Python version as 3-digit string (e.g., '312') matching the cp wheel tag + - name: shortPyVer + type: string + default: '312' + # Target architecture (test stage targets x64 only; ARM64 binaries can't run on x64 host) + - name: architecture + type: string + default: 'x64' + # OneBranch build type: 'Official' or 'NonOfficial' + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Test Both Wheels Windows ${{ parameters.architecture }}' + # Only meaningful when both consolidated artifacts exist in this run. + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Install both wheels + pytest - Windows ${{ parameters.architecture }}' + + # Same custom 1ES pool the Windows build stage uses: Windows Server 2022 with + # SQL Server LocalDB pre-installed, so pytest has a live server without Docker. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 120 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows agents). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + # Download the two CONSOLIDATED wheel artifacts produced earlier in this run. + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python wheels' + inputs: + buildType: 'current' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-wheels' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python-odbc wheels' + inputs: + buildType: 'current' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-odbc-wheels' + + # Start SQL Server LocalDB (pre-installed on the 1ES Windows image). + - powershell: | + sqllocaldb create MSSQLLocalDB + sqllocaldb start MSSQLLocalDB + displayName: 'Start LocalDB instance' + + - powershell: | + sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE DATABASE TestDB" + sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" + sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" + sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" + displayName: 'Setup database and user' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - powershell: | + $ErrorActionPreference = 'Stop' + + # Run every venv-python invocation through this helper so a non-zero exit + # (native commands do NOT trip $ErrorActionPreference) fails the step loudly. + # No param() block: PowerShell then routes ALL tokens (including -m, -c, -q) + # into the automatic $args instead of trying to bind them as parameters. + $script:VenvPy = $null + function Invoke-Py { + & $script:VenvPy @args + if ($LASTEXITCODE -ne 0) { throw "python exited ${LASTEXITCODE}: $($args -join ' ')" } + } + + $PYTAG = 'cp${{ parameters.shortPyVer }}' + $PLATTAG = 'win_amd64' + + # Select the mssql-python wheel matching THIS Python version + platform. + $mpWheel = Get-ChildItem "$(Pipeline.Workspace)\mssql-python-wheels\dist\mssql_python-*$PYTAG-*$PLATTAG*.whl" -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $mpWheel) { throw "ERROR: no mssql-python wheel for $PYTAG/$PLATTAG" } + # The odbc wheel is data-only (py3-none-), one per platform. + $odbcWheel = Get-ChildItem "$(Pipeline.Workspace)\mssql-python-odbc-wheels\dist\mssql_python_odbc-*$PLATTAG*.whl" -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $odbcWheel) { throw "ERROR: no mssql-python-odbc wheel for $PLATTAG" } + + # Fresh isolated venv = clean end-user machine. + $testDir = Join-Path $env:TEMP "enduser_$PYTAG" + if (Test-Path $testDir) { Remove-Item -Recurse -Force $testDir } + python -m venv $testDir + if ($LASTEXITCODE -ne 0) { throw "venv creation failed" } + $script:VenvPy = Join-Path $testDir 'Scripts\python.exe' + + # PYTHONSAFEPATH so the repo checkout can never shadow the installed wheel. + $env:PYTHONSAFEPATH = '1' + + Invoke-Py -m pip install -q -U pip + + # Install the odbc wheel FIRST so mssql-python's pinned mssql-python-odbc==X + # requirement is satisfied from this LOCAL wheel (pip never reaches PyPI). + Write-Host "Installing mssql-python-odbc: $($odbcWheel.Name)" + Invoke-Py -m pip install -q "$($odbcWheel.FullName)" + Write-Host "Installing mssql-python: $($mpWheel.Name)" + Invoke-Py -m pip install -q "$($mpWheel.FullName)" + + # Show where the external ODBC package lives (the split target). + Invoke-Py -c "import mssql_python_odbc, os; d=mssql_python_odbc.get_libs_dir(); print('mssql_python_odbc libs:', d); assert os.path.isdir(d)" + + # POSITIVE assertion: prove the native loader WILL resolve the driver from + # the external mssql_python_odbc package (mirrors GetOdbcLibsBaseDir). + Invoke-Py -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print('external driver path:', drv); assert os.path.exists(drv), 'external ODBC driver not found - loader would NOT select external package: '+drv; print('POSITIVE: loader will resolve the driver from the external mssql_python_odbc package')" + + # PHASE 1: run the FULL existing suite with bundled libs INTACT. Copy the + # suite OUT of the source tree so collection imports the INSTALLED package. + Copy-Item -Recurse "$(Build.SourcesDirectory)\tests" (Join-Path $testDir 'tests') + Copy-Item "$(Build.SourcesDirectory)\pytest.ini" (Join-Path $testDir 'pytest.ini') + Invoke-Py -m pip install -q pytest + if (Test-Path "$(Build.SourcesDirectory)\requirements.txt") { + Invoke-Py -m pip install -q -r "$(Build.SourcesDirectory)\requirements.txt" + } + $env:DB_CONNECTION_STRING = "Server=(localdb)\MSSQLLocalDB;Database=TestDB;Uid=testuser;Pwd=$($env:DB_PASSWORD);TrustServerCertificate=yes" + Push-Location $testDir + try { + Invoke-Py -m pytest (Join-Path $testDir 'tests') -v --maxfail=1 -m "not stress" + } finally { + Pop-Location + } + Write-Host "Full suite passed for $PYTAG with both wheels installed (external driver preferred)" + + # PHASE 2 - DECISIVE negative proof: remove the ODBC driver that + # mssql_python still bundles, then connect again in a FRESH process. + $mpLibs = & $script:VenvPy -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), 'libs'))" + if ($LASTEXITCODE -ne 0) { throw "failed to resolve mssql_python libs dir" } + if ($mpLibs -like "$(Build.SourcesDirectory)*") { throw "ERROR: resolved mssql_python to the source tree ($mpLibs), not the installed wheel" } + if (Test-Path $mpLibs) { + Write-Host "Removing bundled mssql_python/libs to force use of external ODBC package: $mpLibs" + Remove-Item -Recurse -Force $mpLibs + } else { + Write-Host "Note: mssql_python has no bundled libs/ (already split at build time)" + } + + Invoke-Py -c "import mssql_python; print('mssql_python', mssql_python.__version__, 'imported OK (bundled libs removed)')" + Invoke-Py -c "import mssql_python, os; c=mssql_python.connect(os.environ['DB_CONNECTION_STRING']); cur=c.cursor(); cur.execute('SELECT 1'); print('connection OK via EXTERNAL driver:', cur.fetchone()); c.close()" + Write-Host "DECISIVE: connected with bundled libs removed - driver + mssql-auth.dll resolved from external mssql_python_odbc package" + displayName: 'Install both wheels + full pytest + external-only proof' + env: + DB_PASSWORD: $(DB_PASSWORD) From b8f813aa7185335971191195fa3150278154c6a0 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 21 Jul 2026 18:21:18 +0530 Subject: [PATCH 30/35] FIX: suppress DevSkim DS162092 (localhost) findings in macOS both-wheels test stage Convert the two inline DB_CONNECTION_STRING command-prefix env vars (127.0.0.1 = CI SQL Server container) to exported vars on their own line so the '# DevSkim: ignore DS162092' inline suppression applies. Resolves github-advanced-security code-scanning alerts 845 and 846. --- .../stages/wheel-installation-test-macos-stage.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml index 5b923f0fa..0c324df23 100644 --- a/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml +++ b/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml @@ -180,8 +180,10 @@ stages: "$VENV_PY" -m pip install -q -r "$(Build.SourcesDirectory)/requirements.txt" fi cd "$TEST_DIR" - DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ - "$VENV_PY" -m pytest "$TEST_DIR/tests" -v --maxfail=1 -m "not stress" + # 127.0.0.1 targets the CI SQL Server 2022 container on the agent loopback (a + # test fixture), not debug code reaching a real host. + export DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" # DevSkim: ignore DS162092 + "$VENV_PY" -m pytest "$TEST_DIR/tests" -v --maxfail=1 -m "not stress" echo "Full suite passed for ${PYTAG} with both wheels installed (external driver preferred)" # PHASE 2 - DECISIVE negative proof: remove the ODBC driver that @@ -199,8 +201,10 @@ stages: fi "$VENV_PY" -c "import mssql_python; print('mssql_python', mssql_python.__version__, 'imported OK (bundled libs removed)')" - DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ - "$VENV_PY" -c "import mssql_python, os; c=mssql_python.connect(os.environ['DB_CONNECTION_STRING']); cur=c.cursor(); cur.execute('SELECT 1'); print('connection OK via EXTERNAL driver:', cur.fetchone()); c.close()" + # 127.0.0.1 targets the CI SQL Server 2022 container on the agent loopback (a + # test fixture), not debug code reaching a real host. + export DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" # DevSkim: ignore DS162092 + "$VENV_PY" -c "import mssql_python, os; c=mssql_python.connect(os.environ['DB_CONNECTION_STRING']); cur=c.cursor(); cur.execute('SELECT 1'); print('connection OK via EXTERNAL driver:', cur.fetchone()); c.close()" echo "DECISIVE: connected with bundled libs removed - driver resolved from external mssql_python_odbc package" displayName: 'Install both wheels + full pytest + external-only proof' env: From 433a1fb5c20549762ba765d2de4094be8f4d54a1 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 21 Jul 2026 18:29:54 +0530 Subject: [PATCH 31/35] FIX: align Linux both-wheels test stage with the ADO-validated version Replace the PR's hardened-but-unrun Linux both-wheels test flow with the exact stage that was validated green on the ADO nightly pipeline, so shipped == tested. Retains the DS162092 DevSkim suppression on the in-container sqlcmd localhost readiness probe. --- .../stages/wheel-installation-test-stage.yml | 82 +++++++------------ 1 file changed, 29 insertions(+), 53 deletions(-) diff --git a/OneBranchPipelines/stages/wheel-installation-test-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-stage.yml index 3513f4ed1..1fe0c1a62 100644 --- a/OneBranchPipelines/stages/wheel-installation-test-stage.yml +++ b/OneBranchPipelines/stages/wheel-installation-test-stage.yml @@ -11,25 +11,19 @@ # * POSITIVE: before touching anything we assert that the native loader WILL pick # the external package - GetDriverPathCpp(dir(mssql_python_odbc)) points at an # existing driver file, which is exactly the condition GetOdbcLibsBaseDir uses -# to treat the external package as authoritative. Because that resolver prefers -# the external package on EVERY connect whenever its driver is complete - -# regardless of whether mssql_python still bundles libs (see GetOdbcLibsBaseDir -# in ddbc_bindings.cpp) - simply installing the odbc wheel already exercises the -# external-resolution branch throughout the whole suite. -# * NEGATIVE: AFTER the full suite passes, we DELETE the ODBC driver binaries that -# mssql-python still bundles under `mssql_python/libs/` and connect once more. -# With the bundled fallback removed, a successful live query can ONLY be using -# the driver from the separately-installed `mssql_python_odbc` package. We do -# NOT re-run the full suite after deletion: test_000_dependencies intentionally -# asserts the bundled package layout, which this negative step deliberately breaks. +# to treat the external package as authoritative. +# * NEGATIVE: we then DELETE the ODBC driver binaries that mssql-python still +# bundles under `mssql_python/libs/`. With the bundled fallback removed, a +# successful connection + passing test suite proves the driver was resolved +# from the separately-installed `mssql_python_odbc` package. +# (see GetOdbcLibsBaseDir in ddbc_bindings.cpp, which prefers the external package +# and falls back to bundled libs). Without the deletion the bundled fallback would +# mask a broken split and the test would pass even if the ODBC package were ignored. # # Scope: Linux manylinux_2_28 + musllinux (x86_64), all supported Python versions. # This stage tests the INSTALL/RESOLUTION path, not per-arch compilation (already # covered by the build stages). It runs only when BOTH packages are built in the # same run (buildPackage = 'both'), because it needs both consolidated artifacts. -# NOTE: this external-resolution coverage is nightly-only. The PR-validation -# pipeline never installs the odbc wheel, so its connects still use the bundled -# driver path - that gap is intentional and covered here on the daily build. # ========================================================================================= parameters: # Stage identifier (e.g., 'TestBothWheels_manylinux_2_28_x86_64') @@ -72,9 +66,7 @@ stages: name: Python-1ES-pool demands: - imageOverride -equals PYTHON-1ES-UB2404 - # 5 Python versions x full pytest suite (+ venv/pip/install overhead) per - # stage; headroom over the observed runtime so a slow agent cannot time out. - timeoutInMinutes: 150 + timeoutInMinutes: 90 variables: - name: ob_sdl_binskim_enabled @@ -188,8 +180,6 @@ stages: echo "Waiting for SQL Server to be ready..." for i in {1..30}; do - # sqlcmd runs INSIDE the mssql container via `docker exec`; localhost is the - # container's own loopback (a readiness probe), not debug code reaching a dev box. if docker exec sqlserver-$(LINUX_TAG)-$(ARCH) /opt/mssql-tools18/bin/sqlcmd \ -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then # DevSkim: ignore DS162092 echo "SQL Server is ready" @@ -205,9 +195,8 @@ stages: env: DB_PASSWORD: $(DB_PASSWORD) - # For each Python version: install BOTH wheels, run the full pytest suite - # with bundled libs intact (external driver already preferred), THEN remove - # the bundled ODBC driver and connect once more. The post-deletion connect + # For each Python version: install BOTH wheels, remove the bundled ODBC + # driver from the installed mssql_python package, then run pytest. A pass # proves the driver was loaded from the separate mssql_python_odbc package. - script: | set -euxo pipefail @@ -288,34 +277,9 @@ stages: # mssql-auth.dll, which this platform test does not cover). $VENV_PY -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print(\"external driver path:\", drv); assert os.path.exists(drv), \"external ODBC driver not found - loader would NOT select external package: \"+drv; print(\"POSITIVE: loader will resolve the driver from the external mssql_python_odbc package\")" - # PHASE 1: run the FULL existing suite with bundled libs INTACT. - # The external mssql_python_odbc package is already installed, so - # GetOdbcLibsBaseDir() prefers it on every connect (it selects the - # external package whenever its driver is complete, regardless of - # whether mssql_python still bundles libs - see ddbc_bindings.cpp). - # This exercises the external-resolution branch throughout while - # keeping tests that assert the bundled package layout (e.g. - # test_000_dependencies) valid. Copy the suite OUT of /workspace so - # pytest collection cannot import the source-tree package. - cp -r /workspace/tests "$TEST_DIR/tests" - # pytest.ini carries the `-m "not stress"` default and markers; it must - # be present so collection behaves like a normal repo run. Fail loudly - # if it is missing rather than silently running an unconfigured suite. - cp /workspace/pytest.ini "$TEST_DIR/pytest.ini" - $VENV_PY -m pip install -q pytest - if [ -f /workspace/requirements.txt ]; then - $VENV_PY -m pip install -q -r /workspace/requirements.txt - fi - DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ - $VENV_PY -m pytest "$TEST_DIR/tests" -v --maxfail=1 -m "not stress" - echo "Full suite passed for ${PYBIN} with both wheels installed (external driver preferred)" - - # PHASE 2 - DECISIVE negative proof: remove the ODBC driver that - # mssql_python still bundles, then connect again. With the bundled - # fallback gone, a successful live query can ONLY be using the driver - # from the separate mssql_python_odbc package. We do NOT re-run the - # full suite here: test_000_dependencies intentionally asserts the - # bundled layout, which this step deliberately breaks. + # DECISIVE (negative) STEP: remove the ODBC driver bundled inside + # mssql_python. With the fallback gone, any successful connection must + # be using the driver from the separate mssql_python_odbc package. MP_LIBS=$($VENV_PY -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), \"libs\"))") case "$MP_LIBS" in /workspace/*) @@ -328,10 +292,22 @@ stages: echo "Note: mssql_python has no bundled libs/ (already split at build time)" fi - $VENV_PY -c "import mssql_python; print(\"mssql_python\", mssql_python.__version__, \"imported OK (bundled libs removed)\")" + # Smoke test: import + live connection using ONLY the external driver + $VENV_PY -c "import mssql_python; print(\"mssql_python\", mssql_python.__version__, \"imported OK\")" + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -c "import mssql_python, os; c=mssql_python.connect(os.environ[\"DB_CONNECTION_STRING\"]); cur=c.cursor(); cur.execute(\"SELECT 1\"); print(\"connection OK:\", cur.fetchone()); c.close()" + + # Copy the suite OUT of /workspace so pytest collection cannot import + # the source-tree package either, then run against the installed wheels. + cp -r /workspace/tests "$TEST_DIR/tests" + cp /workspace/pytest.ini "$TEST_DIR/pytest.ini" 2>/dev/null || true + $VENV_PY -m pip install -q pytest + if [ -f /workspace/requirements.txt ]; then + $VENV_PY -m pip install -q -r /workspace/requirements.txt + fi DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ - $VENV_PY -c "import mssql_python, os; c=mssql_python.connect(os.environ[\"DB_CONNECTION_STRING\"]); cur=c.cursor(); cur.execute(\"SELECT 1\"); print(\"connection OK via EXTERNAL driver:\", cur.fetchone()); c.close()" - echo "DECISIVE: ${PYBIN} connected with bundled libs removed - driver resolved from external mssql_python_odbc package" + $VENV_PY -m pytest "$TEST_DIR/tests" -v --maxfail=1 + echo "All tests passed for ${PYBIN} using the EXTERNAL mssql_python_odbc driver" ' rc=$? set -e From d10a199fab9d0f75e8804e2df57e52c472134966 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 21 Jul 2026 18:38:30 +0530 Subject: [PATCH 32/35] FIX: align test_000_dependencies with the package-split-aware driver resolution Resolve the expected driver payload via GetOdbcLibsBaseDir (external mssql_python_odbc package when installed, else bundled mssql_python/libs), matching the native runtime resolver PR 664 already ships. Single-wheel CI is unchanged (falls back to bundled libs); the both-wheels test stage that removes bundled libs now passes because test_000 validates the external package's driver. This is the exact test validated green on the ADO nightly pipeline. --- tests/test_000_dependencies.py | 161 ++++++++------------------------- 1 file changed, 40 insertions(+), 121 deletions(-) diff --git a/tests/test_000_dependencies.py b/tests/test_000_dependencies.py index f38ba06e2..e5337c226 100644 --- a/tests/test_000_dependencies.py +++ b/tests/test_000_dependencies.py @@ -7,8 +7,6 @@ import platform import os import re -import shutil -import subprocess import sys from pathlib import Path @@ -22,6 +20,7 @@ def __init__(self): self.platform_name = platform.system().lower() self.raw_architecture = platform.machine().lower() self.module_dir = self._get_module_directory() + self.libs_base_dir = self._get_libs_base_dir() self.normalized_arch = self._normalize_architecture() def _get_module_directory(self): @@ -35,6 +34,33 @@ def _get_module_directory(self): # Fallback to relative path from tests directory return Path(__file__).parent.parent / "mssql_python" + def _get_libs_base_dir(self): + """Directory that contains the ``libs/`` driver payload the loader will use. + + Mirrors the native resolver (``GetOdbcLibsBaseDir`` in ddbc_bindings.cpp): + the external ``mssql_python_odbc`` package is authoritative whenever it is + installed and actually ships the driver; otherwise the driver is resolved + from ``mssql_python``'s own bundled ``libs/``. Keeping the expected paths in + lockstep with the loader means these dependency assertions validate the SAME + driver the runtime loads -- including the package-split scenario where the + bundled ``mssql_python/libs`` is absent and the driver comes only from the + separately installed ``mssql_python_odbc`` package. When that package is not + installed the base is unchanged, so single-wheel CI keeps enforcing that + ``mssql_python`` bundles its own libs. + """ + try: + import mssql_python_odbc + import mssql_python.ddbc_bindings as ddbc + + external_base = Path(mssql_python_odbc.__file__).parent + driver = ddbc.GetDriverPathCpp(str(external_base)) + if driver and Path(driver).exists(): + return external_base + except Exception: + # No external package (or it can't resolve a driver) -> bundled libs. + pass + return self.module_dir + def _normalize_architecture(self): """Normalize architecture names for the given platform.""" arch_lower = self.raw_architecture.lower() @@ -152,7 +178,7 @@ def get_expected_dependencies(self): def _get_windows_dependencies(self): """Get Windows dependencies based on architecture.""" - base_path = self.module_dir / "libs" / "windows" / self.normalized_arch + base_path = self.libs_base_dir / "libs" / "windows" / self.normalized_arch dependencies = [ base_path / self._driver_filename(), @@ -169,7 +195,7 @@ def _get_macos_dependencies(self): # macOS uses universal2 binaries, but we need to check both arch directories for arch in ["arm64", "x86_64"]: - base_path = self.module_dir / "libs" / "macos" / arch / "lib" + base_path = self.libs_base_dir / "libs" / "macos" / arch / "lib" dependencies.extend( [ base_path / self._driver_filename(), @@ -190,7 +216,7 @@ def _get_linux_dependencies(self): elif runtime_arch in ["aarch64"]: runtime_arch = "arm64" - base_path = self.module_dir / "libs" / "linux" / distro_name / runtime_arch / "lib" + base_path = self.libs_base_dir / "libs" / "linux" / distro_name / runtime_arch / "lib" dependencies = [ base_path / self._driver_filename(), @@ -232,7 +258,7 @@ def get_expected_driver_path(self): if platform_name == "windows": driver_path = ( - Path(self.module_dir) + Path(self.libs_base_dir) / "libs" / "windows" / normalized_arch @@ -241,7 +267,7 @@ def get_expected_driver_path(self): elif platform_name == "darwin": driver_path = ( - Path(self.module_dir) + Path(self.libs_base_dir) / "libs" / "macos" / normalized_arch @@ -252,7 +278,7 @@ def get_expected_driver_path(self): elif platform_name == "linux": distro_name = self._detect_linux_distro() driver_path = ( - Path(self.module_dir) + Path(self.libs_base_dir) / "libs" / "linux" / distro_name @@ -356,7 +382,7 @@ class TestArchitectureSpecificDependencies: def test_windows_vcredist_dependency(self): """Test that Windows builds include vcredist dependencies.""" vcredist_path = ( - dependency_tester.module_dir + dependency_tester.libs_base_dir / "libs" / "windows" / dependency_tester.normalized_arch @@ -372,7 +398,7 @@ def test_windows_vcredist_dependency(self): def test_windows_auth_dependency(self): """Test that Windows builds include authentication library.""" auth_path = ( - dependency_tester.module_dir + dependency_tester.libs_base_dir / "libs" / "windows" / dependency_tester.normalized_arch @@ -385,7 +411,7 @@ def test_windows_auth_dependency(self): def test_macos_universal_dependencies(self): """Test that macOS builds include dependencies for both architectures.""" for arch in ["arm64", "x86_64"]: - base_path = dependency_tester.module_dir / "libs" / "macos" / arch / "lib" + base_path = dependency_tester.libs_base_dir / "libs" / "macos" / arch / "lib" msodbcsql_path = base_path / dependency_tester._driver_filename() libodbcinst_path = base_path / "libodbcinst.2.dylib" @@ -395,120 +421,13 @@ def test_macos_universal_dependencies(self): libodbcinst_path.exists() ), f"macOS {arch} ODBC installer library not found: {libodbcinst_path}" - @pytest.mark.skipif(dependency_tester.platform_name != "darwin", reason="macOS-specific test") - def test_macos_driver_load_chain_is_relocatable(self): - """Ensure the macOS driver's bundled load chain contains no absolute paths. - - Guards against GitHub issue #656. On a fresh Apple Silicon Mac (without - ``brew install unixodbc``) importing mssql_python fails because the arm64 - ``libmsodbcsql.18.dylib`` shipped in the wheel still hardcodes - ``/opt/homebrew/lib/libodbcinst.2.dylib`` instead of - ``@loader_path/libodbcinst.2.dylib``. - - Root cause: ``pybind/configure_dylibs.sh`` only rewrites the dylibs for - the *build host* architecture (``ARCH=$(uname -m)``). macOS wheels are - universal2 but built on an x86_64 runner, so only the x86_64 driver gets - relocated; the arm64 driver ships with Homebrew-absolute dependencies. - - At runtime ``ddbc_bindings`` ``dlopen``s ``libmsodbcsql.18.dylib`` - DIRECTLY (it does not go through the unixODBC driver manager), so this - test walks exactly that load graph: starting from the driver and - following only its *bundled* sibling dependencies. Any bundled sibling - reached via an absolute path would fail to load on a machine without - Homebrew. - - ``otool -L`` reads Mach-O load commands of any architecture regardless of - the host, so inspecting BOTH the arm64 and x86_64 drivers lets a single - x86_64 CI runner catch the arm64-only packaging bug. Non-bundled - dependencies (system libs under /usr/lib or /System, and the documented - external openssl prerequisite) are intentionally ignored. - """ - otool = shutil.which("otool") - if otool is None: - pytest.skip("otool not available on this system") - - def direct_bundled_deps(dylib_path, bundled_names): - """Return (absolute_hits, relocatable_hits) for bundled sibling deps. - - The first line of ``otool -L`` output is the file being inspected and - the second is the library's own install id (LC_ID_DYLIB); both are - self-references and are skipped by ignoring deps whose basename equals - the file's own name. - """ - result = subprocess.run([otool, "-L", str(dylib_path)], capture_output=True, text=True) - if result.returncode != 0: - return None, None - - absolute_hits, relocatable_hits = [], [] - for line in result.stdout.splitlines()[1:]: - dep = line.strip().split(" (compatibility")[0].strip() - if not dep: - continue - base = os.path.basename(dep) - if base == dylib_path.name or base not in bundled_names: - continue # self-reference or a non-bundled/external dependency - if dep.startswith("/"): - absolute_hits.append((base, dep)) - else: - relocatable_hits.append(base) - return absolute_hits, relocatable_hits - - problems = [] - checked_arches = [] - for arch in ["arm64", "x86_64"]: - lib_dir = dependency_tester.module_dir / "libs" / "macos" / arch / "lib" - driver = lib_dir / "libmsodbcsql.18.dylib" - if not driver.exists(): - continue - checked_arches.append(arch) - - bundled = {p.name: p for p in lib_dir.glob("*.dylib")} - - # Depth-first walk of the driver's bundled load chain (queue.pop() is - # LIFO). Order does not matter here: we visit every reachable bundled - # dylib and collect all absolute-path problems regardless of traversal. - visited, queue = set(), [driver] - while queue: - current = queue.pop() - if current.name in visited: - continue - visited.add(current.name) - - absolute_hits, relocatable_hits = direct_bundled_deps(current, bundled) - if absolute_hits is None or relocatable_hits is None: - problems.append(f"{arch}/{current.name}: otool failed to inspect the library") - continue - - for base, dep in absolute_hits: - problems.append( - f"{arch}/{current.name} loads bundled '{base}' via absolute " - f"path '{dep}' (expected @loader_path); this breaks on " - f"machines without Homebrew" - ) - for base in relocatable_hits: - queue.append(bundled[base]) - - # Fail loudly instead of passing vacuously if no bundled driver was found - # to validate (e.g. the packaging layout changed). - assert checked_arches, ( - "no bundled macOS driver found to validate under " - "libs/macos/{arm64,x86_64}/lib/libmsodbcsql.18.dylib " - "(packaging layout may have changed)" - ) - - assert not problems, ( - "macOS driver load chain contains hardcoded absolute dependency paths " - "that break on machines without Homebrew (see GitHub issue #656):\n " - + "\n ".join(problems) - ) - @pytest.mark.skipif(dependency_tester.platform_name != "linux", reason="Linux-specific test") def test_linux_distribution_dependencies(self): """Test that Linux builds include distribution-specific dependencies.""" distro_name = dependency_tester._detect_linux_distro() # Test that the distribution directory exists - distro_path = dependency_tester.module_dir / "libs" / "linux" / distro_name + distro_path = dependency_tester.libs_base_dir / "libs" / "linux" / distro_name assert distro_path.exists(), f"Linux distribution directory not found: {distro_path}" @@ -583,9 +502,9 @@ def test_get_driver_path_from_ddbc_bindings(): try: import mssql_python.ddbc_bindings as ddbc - module_dir = dependency_tester.module_dir + libs_base = dependency_tester.libs_base_dir - driver_path = ddbc.GetDriverPathCpp(str(module_dir)) + driver_path = ddbc.GetDriverPathCpp(str(libs_base)) # The driver path should be same as one returned by the Python function expected_path = dependency_tester.get_expected_driver_path() From e2647e36f743146ea860119f5a53668357a6d93c Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 21 Jul 2026 18:54:36 +0530 Subject: [PATCH 33/35] FIX: strengthen odbc-package build-stage wheel-content guards (ADO parity) Assert the built mssql-python-odbc wheel packages the full driver payload, not just the driver DLL: libodbcinst on Linux/macOS and mssql-auth.dll on Windows. The native loader treats the external package as complete only when these companions are co-located; without them it would silently fall back to bundled libs or fail. Matches the guards validated on the ADO nightly pipeline. --- .../stages/build-odbc-linux-stage.yml | 12 +++++++++--- .../stages/build-odbc-macos-stage.yml | 12 +++++++++--- .../stages/build-odbc-windows-stage.yml | 18 +++++++++++++----- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml index 969c0b8b3..51da9144b 100644 --- a/OneBranchPipelines/stages/build-odbc-linux-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -149,12 +149,18 @@ stages: fi for whl in $matches; do echo "Verifying ODBC driver binary present in: $whl" - if python3 -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then - echo "OK: $whl contains the ODBC driver binary" - else + entries=$(python3 -m zipfile -l "$whl") + if ! echo "$entries" | grep -qi 'msodbcsql'; then echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 exit 1 fi + # The msodbcsql driver loads the ODBC installer lib at runtime; assert + # the companion is packaged too, not just the driver itself. + if ! echo "$entries" | grep -qi 'libodbcinst'; then + echo "ERROR: ODBC installer lib (libodbcinst*) missing from $whl" >&2 + exit 1 + fi + echo "OK: $whl contains the ODBC driver binary and libodbcinst" done displayName: 'Assert wheel contains ODBC driver binary' diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml index c1e7541b1..7378bc868 100644 --- a/OneBranchPipelines/stages/build-odbc-macos-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -91,12 +91,18 @@ stages: fi for whl in $matches; do echo "Verifying ODBC driver binary present in: $whl" - if python -m zipfile -l "$whl" | grep -qi 'msodbcsql'; then - echo "OK: $whl contains the ODBC driver binary" - else + entries=$(python -m zipfile -l "$whl") + if ! echo "$entries" | grep -qi 'msodbcsql'; then echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 exit 1 fi + # The msodbcsql driver loads the ODBC installer lib at runtime; assert + # the companion is packaged too, not just the driver itself. + if ! echo "$entries" | grep -qi 'libodbcinst'; then + echo "ERROR: ODBC installer lib (libodbcinst*) missing from $whl" >&2 + exit 1 + fi + echo "OK: $whl contains the ODBC driver binary and libodbcinst" done displayName: 'Assert wheel contains ODBC driver binary' diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml index dea809ed0..3fc9903b8 100644 --- a/OneBranchPipelines/stages/build-odbc-windows-stage.yml +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -92,8 +92,12 @@ stages: TargetFolder: '$(ob_outputDirectory)\wheels' displayName: 'Stage wheel artifact' - # Guard against a data wheel that packaged no driver binary (twine check - # would still pass). The driver DLL name contains 'msodbcsql' on every platform. + # Guard against a data wheel that packaged an incomplete driver set (twine + # check would still pass). Assert BOTH the driver DLL (msodbcsql*) and the + # Windows auth component (mssql-auth.dll): the native loader treats the + # external package as COMPLETE on Windows only when mssql-auth.dll is + # co-located with the driver - without it the loader silently falls back to + # bundled libs (or fails), which the Linux-only install test cannot catch. - powershell: | $ErrorActionPreference = "Stop" $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" @@ -104,13 +108,17 @@ stages: Add-Type -AssemblyName System.IO.Compression.FileSystem foreach ($whl in $wheels) { $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) - $hasDriver = $zip.Entries | Where-Object { $_.FullName -match 'msodbcsql' } + $names = $zip.Entries | ForEach-Object { $_.FullName } $zip.Dispose() - if (-not $hasDriver) { + if (-not ($names | Where-Object { $_ -match 'msodbcsql' })) { Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" exit 1 } - Write-Host "OK: $($whl.Name) contains the ODBC driver binary" + if (-not ($names | Where-Object { $_ -match 'mssql-auth\.dll' })) { + Write-Error "Windows auth component (mssql-auth.dll) missing from $($whl.Name) - loader would not treat the external package as complete" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary and mssql-auth.dll" } displayName: 'Assert wheel contains ODBC driver binary' From f23df6963c75a02430453351ac7b4af05f965786 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 21 Jul 2026 19:13:50 +0530 Subject: [PATCH 34/35] FIX: restore GitHub #656 macOS driver relocatability test under package split The split-aware test_000 rewrite dropped test_macos_driver_load_chain_is_relocatable, the regression guard for GitHub #656 (arm64 libmsodbcsql.18.dylib hardcoding /opt/homebrew/lib/libodbcinst.2.dylib instead of @loader_path, breaking import on Macs without Homebrew). Restore it, repointed module_dir -> libs_base_dir so it validates the driver the loader actually resolves (bundled or external mssql_python_odbc). otool inspects both arm64 and x86_64 slices, so one x86_64 runner catches the arm64-only packaging bug. --- tests/test_000_dependencies.py | 109 +++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/tests/test_000_dependencies.py b/tests/test_000_dependencies.py index e5337c226..02784181a 100644 --- a/tests/test_000_dependencies.py +++ b/tests/test_000_dependencies.py @@ -7,6 +7,8 @@ import platform import os import re +import shutil +import subprocess import sys from pathlib import Path @@ -421,6 +423,113 @@ def test_macos_universal_dependencies(self): libodbcinst_path.exists() ), f"macOS {arch} ODBC installer library not found: {libodbcinst_path}" + @pytest.mark.skipif(dependency_tester.platform_name != "darwin", reason="macOS-specific test") + def test_macos_driver_load_chain_is_relocatable(self): + """Ensure the macOS driver's bundled load chain contains no absolute paths. + + Guards against GitHub issue #656. On a fresh Apple Silicon Mac (without + ``brew install unixodbc``) importing mssql_python fails because the arm64 + ``libmsodbcsql.18.dylib`` shipped in the wheel still hardcodes + ``/opt/homebrew/lib/libodbcinst.2.dylib`` instead of + ``@loader_path/libodbcinst.2.dylib``. + + Root cause: ``pybind/configure_dylibs.sh`` only rewrites the dylibs for + the *build host* architecture (``ARCH=$(uname -m)``). macOS wheels are + universal2 but built on an x86_64 runner, so only the x86_64 driver gets + relocated; the arm64 driver ships with Homebrew-absolute dependencies. + + At runtime ``ddbc_bindings`` ``dlopen``s ``libmsodbcsql.18.dylib`` + DIRECTLY (it does not go through the unixODBC driver manager), so this + test walks exactly that load graph: starting from the driver and + following only its *bundled* sibling dependencies. Any bundled sibling + reached via an absolute path would fail to load on a machine without + Homebrew. + + ``otool -L`` reads Mach-O load commands of any architecture regardless of + the host, so inspecting BOTH the arm64 and x86_64 drivers lets a single + x86_64 CI runner catch the arm64-only packaging bug. Non-bundled + dependencies (system libs under /usr/lib or /System, and the documented + external openssl prerequisite) are intentionally ignored. + """ + otool = shutil.which("otool") + if otool is None: + pytest.skip("otool not available on this system") + + def direct_bundled_deps(dylib_path, bundled_names): + """Return (absolute_hits, relocatable_hits) for bundled sibling deps. + + The first line of ``otool -L`` output is the file being inspected and + the second is the library's own install id (LC_ID_DYLIB); both are + self-references and are skipped by ignoring deps whose basename equals + the file's own name. + """ + result = subprocess.run([otool, "-L", str(dylib_path)], capture_output=True, text=True) + if result.returncode != 0: + return None, None + + absolute_hits, relocatable_hits = [], [] + for line in result.stdout.splitlines()[1:]: + dep = line.strip().split(" (compatibility")[0].strip() + if not dep: + continue + base = os.path.basename(dep) + if base == dylib_path.name or base not in bundled_names: + continue # self-reference or a non-bundled/external dependency + if dep.startswith("/"): + absolute_hits.append((base, dep)) + else: + relocatable_hits.append(base) + return absolute_hits, relocatable_hits + + problems = [] + checked_arches = [] + for arch in ["arm64", "x86_64"]: + lib_dir = dependency_tester.libs_base_dir / "libs" / "macos" / arch / "lib" + driver = lib_dir / "libmsodbcsql.18.dylib" + if not driver.exists(): + continue + checked_arches.append(arch) + + bundled = {p.name: p for p in lib_dir.glob("*.dylib")} + + # Depth-first walk of the driver's bundled load chain (queue.pop() is + # LIFO). Order does not matter here: we visit every reachable bundled + # dylib and collect all absolute-path problems regardless of traversal. + visited, queue = set(), [driver] + while queue: + current = queue.pop() + if current.name in visited: + continue + visited.add(current.name) + + absolute_hits, relocatable_hits = direct_bundled_deps(current, bundled) + if absolute_hits is None or relocatable_hits is None: + problems.append(f"{arch}/{current.name}: otool failed to inspect the library") + continue + + for base, dep in absolute_hits: + problems.append( + f"{arch}/{current.name} loads bundled '{base}' via absolute " + f"path '{dep}' (expected @loader_path); this breaks on " + f"machines without Homebrew" + ) + for base in relocatable_hits: + queue.append(bundled[base]) + + # Fail loudly instead of passing vacuously if no bundled driver was found + # to validate (e.g. the packaging layout changed). + assert checked_arches, ( + "no bundled macOS driver found to validate under " + "libs/macos/{arm64,x86_64}/lib/libmsodbcsql.18.dylib " + "(packaging layout may have changed)" + ) + + assert not problems, ( + "macOS driver load chain contains hardcoded absolute dependency paths " + "that break on machines without Homebrew (see GitHub issue #656):\n " + + "\n ".join(problems) + ) + @pytest.mark.skipif(dependency_tester.platform_name != "linux", reason="Linux-specific test") def test_linux_distribution_dependencies(self): """Test that Linux builds include distribution-specific dependencies.""" From 67ecc8629daba1268703a89b16d25919748d9040 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Tue, 21 Jul 2026 19:17:38 +0530 Subject: [PATCH 35/35] FIX: make mssql-python-odbc pin guard non-blocking and anchor the version Per review: the release-pipeline pin guard hard-failed every mssql-python release because setup.py is still pre-pin (declares only azure-identity), and its unanchored regex let 18.6.20 satisfy an 18.6.2 pin. Make it non-blocking - a wheel with no mssql-python-odbc dependency (pre-pin build) warns-and-skips instead of failing, so an unrelated hotfix released before the pin lands is not blocked. When a wheel DOES declare the dependency, an anchored regex (negative lookahead (?![\d.])) enforces the exact pin and rejects a wrong odbc version. Applies to both dummy and official release pipelines. --- OneBranchPipelines/dummy-release-pipeline.yml | 32 ++++++++++++------- .../official-release-pipeline.yml | 32 ++++++++++++------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 6cbb7c44b..ac0c38666 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -205,11 +205,13 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' - # Guard: the released mssql-python wheel MUST depend on the pinned - # mssql-python-odbc package. Catches releasing from a PRE-pin build - # (e.g. the wrong build run), which would ship a driver-less package - # that breaks 'pip install mssql-python'. By design this FAILS on a - # pre-pin build. Update $expectedVersion when the odbc pin bumps. + # Guard: if the released mssql-python wheel declares an + # mssql-python-odbc dependency, it MUST be the exact pinned version. + # A pre-pin build declares no such dependency and is allowed through + # (inert until setup.py adds install_requires), so this does not break + # an unrelated hotfix released before the pin lands. It DOES catch a + # wheel pinned to the wrong odbc version. Update $expectedVersion when + # the odbc pin bumps. - task: PowerShell@2 displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' inputs: @@ -237,16 +239,22 @@ extends: $reader.Dispose() $zip.Dispose() $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } - if ($odbcLines) { + if (-not $odbcLines) { + Write-Warning "Wheel declares no mssql-python-odbc dependency (pre-pin build). Skipping odbc pin enforcement - expected until setup.py adds install_requires=mssql-python-odbc==$expectedVersion." + } else { Write-Host "odbc dependency in metadata:" $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + # Anchor the version with a negative lookahead so 18.6.20 / + # 18.6.2.1 do NOT satisfy an 18.6.2 pin, while still allowing a + # trailing space, ';' environment marker, or end of line. + $pinRegex = 'mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + '(?![\d.])' + $pinned = $odbcLines | Where-Object { $_ -match $pinRegex } + if (-not $pinned) { + Write-Error "mssql-python wheel declares an mssql-python-odbc dependency but not the exact pin ==$expectedVersion. Release from the POST-pin build with the correct odbc version." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" } - $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) - if ($metadata -notmatch $pinRegex) { - Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." - exit 1 - } - Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" # Step 4: Verify wheel integrity - task: PowerShell@2 diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 8937fd0f6..43cc32d68 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -205,11 +205,13 @@ extends: filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' - # Guard: the released mssql-python wheel MUST depend on the pinned - # mssql-python-odbc package. Catches releasing from a PRE-pin build - # (e.g. the wrong build run), which would ship a driver-less package - # that breaks 'pip install mssql-python'. By design this FAILS on a - # pre-pin build. Update $expectedVersion when the odbc pin bumps. + # Guard: if the released mssql-python wheel declares an + # mssql-python-odbc dependency, it MUST be the exact pinned version. + # A pre-pin build declares no such dependency and is allowed through + # (inert until setup.py adds install_requires), so this does not break + # an unrelated hotfix released before the pin lands. It DOES catch a + # wheel pinned to the wrong odbc version. Update $expectedVersion when + # the odbc pin bumps. - task: PowerShell@2 displayName: 'Validate mssql-python-odbc pin in wheel metadata' inputs: @@ -237,16 +239,22 @@ extends: $reader.Dispose() $zip.Dispose() $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } - if ($odbcLines) { + if (-not $odbcLines) { + Write-Warning "Wheel declares no mssql-python-odbc dependency (pre-pin build). Skipping odbc pin enforcement - expected until setup.py adds install_requires=mssql-python-odbc==$expectedVersion." + } else { Write-Host "odbc dependency in metadata:" $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + # Anchor the version with a negative lookahead so 18.6.20 / + # 18.6.2.1 do NOT satisfy an 18.6.2 pin, while still allowing a + # trailing space, ';' environment marker, or end of line. + $pinRegex = 'mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + '(?![\d.])' + $pinned = $odbcLines | Where-Object { $_ -match $pinRegex } + if (-not $pinned) { + Write-Error "mssql-python wheel declares an mssql-python-odbc dependency but not the exact pin ==$expectedVersion. Release from the POST-pin build with the correct odbc version." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" } - $pinRegex = 'Requires-Dist:\s*mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) - if ($metadata -notmatch $pinRegex) { - Write-Error "mssql-python wheel does not pin mssql-python-odbc==$expectedVersion. Are you releasing from the POST-pin build? mssql-python must depend on the pinned odbc package before release." - exit 1 - } - Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" # Step 4: Verify wheel integrity - task: PowerShell@2