From 718fff089f42da7a5d5d69d8334c149006a38c5c Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:26:47 +0530 Subject: [PATCH 01/12] FIX: Enforce bundled native compatibility in Conda packages Port focused native inventory, architecture, ABI and glibc guards; retain the required core and independently probe its load. Preserve newer main cross-build behavior and existing RPATHs. Move the six existing-file prerequisites out of the dependent release layer, including local SQL CI login permissions and test dependencies. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../scripts/build_conda_packages.py | 42 +++-- README.md | 59 ++++++- conda/README.md | 39 +++++ conda/mssql-python/bld.bat | 16 +- conda/mssql-python/build.sh | 4 +- conda/mssql-python/meta.yaml | 6 +- eng/pipelines/pr-validation-pipeline.yml | 6 + eng/requirements-build-macos.txt | 75 +++++++++ eng/requirements-test-linux.txt | 75 +++++++++ eng/scripts/_conda_pkg.py | 67 +++++++- eng/scripts/assert_macho_arch.py | 13 +- eng/scripts/assert_pe_machine.py | 11 +- eng/scripts/audit_bundled_binaries.py | 145 +++++++++++++--- requirements.txt | 1 + tests/test_029_bundled_binary_audit.py | 155 +++++++++++++++++- tests/test_030_pe_machine_assert.py | 39 ++++- tests/test_034_conda_verify_cwd.py | 107 +++++++++++- tests/test_035_conda_macho_assert.py | 22 ++- 18 files changed, 816 insertions(+), 66 deletions(-) create mode 100644 conda/README.md diff --git a/OneBranchPipelines/scripts/build_conda_packages.py b/OneBranchPipelines/scripts/build_conda_packages.py index 45a3d63b..d8d79c3c 100644 --- a/OneBranchPipelines/scripts/build_conda_packages.py +++ b/OneBranchPipelines/scripts/build_conda_packages.py @@ -4,7 +4,7 @@ Replaces build-conda-packages.ps1 + build-conda-packages.sh (the same 7-step pipeline written twice, which had already drifted). conda is Python and every agent has a bootstrap interpreter, so ONE orchestrator runs on every leg; the platform differences (the Miniforge -installer, the win-arm64 Terms-of-Service auto-accept, the Linux-only reachability gate) are +installer, the win-arm64 channel profile, the Linux-only reachability gate) are a handful of branches, not a second 360-line script. Running as a NORMAL process also means the caller reads the exit code directly -- so the PowerShell ErrorActionPreference flips, the `2>$null` swallows, and the `cmd /c "exit 0"` reset all disappear. @@ -306,12 +306,6 @@ def build_env( # host can execute it (natively / Rosetta 2 / QEMU binfmt). env["CONDA_SUBDIR"] = cross_target_subdir _log(f"Cross-targeting conda subdir: CONDA_SUBDIR={cross_target_subdir}") - if cross_target_subdir == "win-arm64": - # win-arm64 deps (python 3.12-3.14, cryptography, vc14_runtime, pyodbc) live on - # Anaconda `defaults`, not conda-forge. Auto-accept the defaults ToS so the - # unattended host-env + verify solves never block on a prompt. - env["CONDA_PLUGINS_AUTO_ACCEPT_TOS"] = "yes" - _log("win-arm64: CONDA_PLUGINS_AUTO_ACCEPT_TOS=yes") if cross_target_subdir.endswith("aarch64") and os.path.isdir("/usr/aarch64-linux-gnu"): # Emulated aarch64 verify runs under qemu-user; point it at the aarch64 glibc loader. env.setdefault("QEMU_LD_PREFIX", "/usr/aarch64-linux-gnu") @@ -392,17 +386,16 @@ def audit_packages( env=env, what="RUNPATH self-containment audit", ) - # win-arm64 is cross-built on x64 where its runtime import is skipped, so its arch is - # trusted from the wheel filename UNLESS the PE machine assert reads it out of the payload. - if target_subdir == "win-arm64": + # Both Windows packages must retain the core; cross builds also rely on static architecture. + if target_subdir in ("win-64", "win-arm64"): pe = os.path.join(eng, "assert_pe_machine.py") if not os.path.isfile(pe): _die(f"PE machine-type assert script not found at {pe}") - _log("=== win-arm64 PE machine-type assert (vendored .pyd/.dll must be ARM64) ===") + _log(f"=== {target_subdir} PE machine-type and required native-component assert ===") run( - [conda, "run", "-n", builder, "python", pe, "--root", bld, "--subdir", "win-arm64"], + [conda, "run", "-n", builder, "python", pe, "--root", bld, "--subdir", target_subdir], env=env, - what="win-arm64 PE machine-type assert", + what=f"{target_subdir} PE machine-type assert", ) # osx legs: verify the universal binding contains the target slice and each thin vendored # driver dylib matches its architecture-specific directory. osx-arm64 is cross-built on the @@ -470,14 +463,27 @@ def _import_probe(mod_name: str, ok_label: str) -> str: INSTALLED package, not the checkout. Uses abspath (NOT realpath) so conda's softlink install mode -- where the site-packages entry symlinks into the pkgs/ cache OUTSIDE the prefix -- is not false-failed: the import PATH stays under the prefix regardless of hard/soft link; only - the symlink TARGET would not. Then prints ok_label + the version.""" + the symlink TARGET would not. Then prints ok_label + the installed module path.""" return ( f"import os,sys,{mod_name} as m;" "f=os.path.normcase(os.path.abspath(m.__file__));" "pref=os.path.normcase(os.path.abspath(sys.prefix));" f"assert f.startswith(pref+os.sep),{mod_name!r}+' loaded from '+m.__file__+" "', not under the conda env '+sys.prefix+' (stray PYTHONPATH/.pth?)';" - f"print({ok_label!r},m.__version__)" + f"print({ok_label!r},m.__file__)" + ) + + +def _core_probe() -> str: + return ( + _import_probe("mssql_py_core", "CORE_PACKAGE_OK") + ";import importlib.machinery;" + "exts=[v for k,v in list(sys.modules.items()) " + "if (k=='mssql_py_core' or k.startswith('mssql_py_core.')) " + "and isinstance(getattr(v,'__loader__',None),importlib.machinery.ExtensionFileLoader)];" + "assert exts,'mssql_py_core did not load its required native extension';" + "assert all(os.path.normcase(os.path.abspath(v.__file__)).startswith(pref+os.sep) " + "for v in exts),'core native extension loaded outside installed prefix';" + "print('CORE_NATIVE_OK',*[v.__file__ for v in exts])" ) @@ -610,6 +616,12 @@ def _verify_impl( f"arm64 cross-build. Refusing to silently skip validation. Output: {out}" ) + # A separate process prevents API/driver preloads from masking core load failures. + run( + [conda, "run", "-n", name, "python", "-c", _core_probe()], + env=env, + what=f"independent required mssql_py_core load (py {py})", + ) _log(f"=== [py {py}] import mssql_python + prove the vendored ODBC payload is present ===") run( [ diff --git a/README.md b/README.md index d24937dc..ce44847d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The driver is compatible with all the Python versions >= 3.10 > **Important Note:** > > ### ODBC Driver Distribution -> The ODBC driver binaries used by `mssql-python` are distributed exclusively through a dedicated companion package: +> For **pip/PyPI installations**, the ODBC driver binaries used by `mssql-python` are distributed through a dedicated companion distribution: > > - Package: `mssql-python-odbc` > - Import name: `mssql_python_odbc` @@ -20,12 +20,18 @@ The driver is compatible with all the Python versions >= 3.10 > > `mssql-python` depends on `mssql-python-odbc==18.6.2.1`. The ODBC driver is loaded lazily when the first connection is created. `pip install mssql-python` transparently pulls the companion package alongside it — no separate install step is required. > -> Starting with v1.13.0, the bundled `libs/` fallback that shipped in v1.12.0 has been removed. Creating a connection will fail if `mssql-python-odbc` is not installed. If you install `mssql-python` from a private index or with `--no-deps`, make sure `mssql-python-odbc==18.6.2.1` is installed alongside it. +> Starting with v1.13.0, the bundled `libs/` fallback that shipped in v1.12.0 has been removed. For pip installations, creating a connection will fail if `mssql-python-odbc` is not installed. If you install `mssql-python` from a private index or with `--no-deps`, make sure `mssql-python-odbc==18.6.2.1` is installed alongside it. +> +> The **temporary Conda candidate** instead combines the code and ODBC payload in one `mssql-python` Conda package. It does not require a separately installed Conda ODBC package. This is not an announcement of public channel availability or release qualification; see the [Conda installation, migration, and readiness guide](conda/README.md). > > ### ODBC Provider Selection (opt-in) > `mssql-python` also supports selecting an alternate native ODBC provider before the first connection, via the `mssql_python.native_provider` module property or the `MSSQL_PYTHON_NATIVE_PROVIDER` environment variable (which takes precedence). A conflicting property assignment emits a `RuntimeWarning`. The default, `"msodbcsql18"`, is unchanged; opting into `"mssql-odbc"` requires the `mssql-python-rs` package (which bundles the Rust ODBC driver alongside the Rust TDS core). Call `mssql_python.get_native_provider_info()` to check the selected provider, source, package version, and resolved driver path. ## Installation + +The pip commands below describe the public release. The temporary combined Conda +candidate has a separate platform matrix and Linux compatibility floor; it is not +covered by the public release's production-readiness statement above. **Windows:** mssql-python can be installed with [pip](http://pypi.python.org/pypi/pip) ```bash @@ -60,6 +66,55 @@ tdnf distro-sync && tdnf install -y libtool-ltdl krb5-libs glibc-iconv pip install mssql-python ``` +**Conda candidate:** Obtain the exact candidate channel and version from its owner; +these changes do not publish packages to the public `microsoft` channel. The candidate +includes the ODBC Driver 18 payload and required bulk-copy core. Linux requires +**glibc >=2.34** for that complete payload; `krb5`, OpenSSL, and `libltdl` resolve from +`conda-forge`, so the system package steps above are not required. Windows uses SChannel. +On macOS, encrypted connections still require system OpenSSL from Homebrew +(`brew install openssl`) or MacPorts, not Conda OpenSSL. Windows ARM64 dependencies +resolve from `defaults`; handle channel terms separately, without automatic acceptance. +Use a fresh environment and replace the placeholders with the owner-provided values: +```bash +# Windows x64, macOS, and Linux +conda install -c "" -c microsoft -c conda-forge --strict-channel-priority --override-channels "mssql-python=" + +# Windows ARM64 +conda install -c "" -c microsoft -c defaults --override-channels "mssql-python=" +``` + +**Conda release maintainers:** The dependent [release-additions PR](https://github.com/microsoft/mssql-python/pull/720) +supplies `OneBranchPipelines/conda-release-pipeline.yml` and the publication/provenance +validators; they are not part of this native-packaging change. The workflow described +below requires those additions and does not establish that production setup or publication +has occurred. Its default is `publishToConda=false`. +Select the exact completed Conda build and expected package version. +The pipeline verifies its recorded upstream wheel run, checks the 28-package matrix and +ELF/PE/Mach-O payloads, and logs archive SHA-256 values and the upload plan without publishing. +Feature-branch candidates are allowed only for validation; production requires release, +Conda producer, and wheel sources on `refs/heads/main`. Recipe and wheel commits may differ, +but each must match its authoritative run record. This is static release readiness, not +live SQL/TLS, bulk-copy, or Arrow feature certification. + +**Production prerequisite (administrator setup, not performed by a dry run):** in the +`SqlClientDrivers/mssql-python` ADO project, protect the existing **Anaconda Publishing** +variable group **117** with an enabled native **Exclusive lock** check and a designated +**Approval** check. Authorize only release definition **2322**, and grant its build identity +read access to group/check configuration and run-check evidence. Record the installed check +IDs as non-secret group variables `CONDA_PUBLICATION_LOCK_CHECK_ID` and +`CONDA_PUBLICATION_APPROVAL_CHECK_ID`; absent or mismatched IDs block publication. Every writer to +`microsoft/mssql-python` label `main` must use this same protected resource; other credentials +or pipelines must not bypass it. The dependent workflow's publish-only `CondaRelease` stage references this group +with `lockBehavior: sequential`, and refuses upload or promotion without matching, +successful current-stage checks. YAML `lockBehavior` alone does not create the lock. +The server-enforced stage lock must cover snapshot, upload, promotion, rollback, and cleanup. +Rollback is compensating, not atomic; a killed process can leave partial labels, which a +subsequent authorized run must re-verify. Metadata/label API calls use 15-second connect +and 60-second read timeouts; the publishing job, including CLI uploads, is capped at 60 minutes. +Until resource setup and a controlled positive +lock/approval evaluation are verified, production remains blocked; validate-only success +does not prove or authorize production publication. + ## Key Features ### Supported Platforms diff --git a/conda/README.md b/conda/README.md new file mode 100644 index 00000000..5b2251eb --- /dev/null +++ b/conda/README.md @@ -0,0 +1,39 @@ +# Combined Conda candidate + +This recipe combines the matching code and ODBC wheels into **one `mssql-python` +Conda package**, including the required bulk-copy core. Pip instead installs +`mssql-python-odbc` as a separate companion distribution. Neither requires a +separately installed ODBC driver or driver manager. + +This is a temporary candidate, not an announcement of public channel availability. +Obtain the exact candidate archive/channel from its owner and install into a new +Conda environment. Activate it and select the same interpreter/kernel in your IDE +or notebook. Application imports and the public API remain unchanged. Use Conda +for upgrades; do not overwrite Conda-owned driver files with pip. + +Linux requires **glibc >=2.34** for the complete native payload, including the +bulk-copy core, even when the binding wheel has a lower platform tag. Do not force +installation on older glibc. This does not change the separate PyPI support claim. +macOS retains its external Homebrew/MacPorts OpenSSL prerequisite for encryption; +Conda OpenSSL alone does not satisfy the driver's system-path lookup. + +The release goal is the matching PyPI release's public API and supported feature +behavior on all 28 ordinary CPython variants: 3.10-3.14 on win-64, linux-64, +linux-aarch64, osx-64 and osx-arm64; 3.12-3.14 on win-arm64. No silent removal of +required native functionality is acceptable. Optional features require their +corresponding dependencies; Windows ARM64 PyArrow availability remains a blocker +to qualifying those features, not permission to drop them. + +These guards are **not full-matrix parity certification**. Native target execution, +SQL, certificate-verified TLS, authentication, bulk-copy and optional-feature tests +remain required before release. Cross-build/static checks and a DB-less driver +load do not establish those results. Applicable OS, certificate and authentication +configuration remain external prerequisites. + +Use only organizationally approved channels and handle applicable terms separately; +the Windows ARM64 dependency profile includes Anaconda `defaults`. Run this existing +build workflow only in a disposable isolated installation: shared-environment +ownership hardening is outside this change. The dependent release-additions PR supplies +the proposed publication/provenance gates; see the [release-maintainer prerequisites](../README.md#installation). +Neither this native-packaging change nor validate-only success authorizes production +publication. diff --git a/conda/mssql-python/bld.bat b/conda/mssql-python/bld.bat index 6e9991fd..91dbc8e3 100644 --- a/conda/mssql-python/bld.bat +++ b/conda/mssql-python/bld.bat @@ -39,17 +39,11 @@ if errorlevel 1 ( echo ERROR: extracted "!CODE_WHL!" has no mssql_python\ddbc_bindings.cp%CONDA_PY% pyd ^(wrong-Python binding^). exit /b 1 ) - REM Keep mssql_py_core when the wheel provides a matching-arch native ext so bulk copy - REM ships (PR #737 makes the win-arm64 wheel vendor the arm64 core). If only the legacy - REM x64 core is present (a pre-#737 wheel), strip it so the package never carries a core - REM that can't load on the target -- the .pyd name encodes the arch. Bulk copy then lazily - REM reports "not available"; the rest of the DBAPI works. Mirrors the ddbc check above. - if exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" ( - echo Keeping matching-arch mssql_py_core; bulk copy enabled on the !ODBC_ARCH! package. - ) else ( - echo No cp%CONDA_PY%-!ODBC_ARCH! mssql_py_core in the wheel; removing the mismatched core ^(bulk copy unavailable until the arm64-core wheel ships^). - if exist "%SP%\mssql_py_core" rmdir /s /q "%SP%\mssql_py_core" - if exist "%SP%\mssql_py_core.libs" rmdir /s /q "%SP%\mssql_py_core.libs" + REM Never silently drop bulk copy. Bare .pyd is the Windows stable-ABI suffix; + REM the package's PE audit checks actual architecture before staging. + if not exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" if not exist "%SP%\mssql_py_core\mssql_py_core.pyd" ( + echo ERROR: required mssql_py_core is missing or incompatible with cp%CONDA_PY% !ODBC_ARCH!. Use a corrected upstream wheel; refusing reduced functionality. + exit /b 1 ) ) else ( "%PYTHON%" -m pip install --no-deps --no-index --find-links "%WHEELS_DIR%" %PKG_NAME%==%PKG_VERSION% -vv diff --git a/conda/mssql-python/build.sh b/conda/mssql-python/build.sh index 355b1a6b..2bf1733d 100644 --- a/conda/mssql-python/build.sh +++ b/conda/mssql-python/build.sh @@ -26,7 +26,9 @@ else # universal2 wheels are cpXY-specific (compiled ddbc_bindings), so filter on the # target CONDA_PY to never grab another interpreter's wheel (mirrors bld.bat). code_whl="" - for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-cp${CONDA_PY}-"*.whl; do + # Match macosx explicitly (like the odbc glob below) so a stray Linux cpXY wheel staged in + # the same dir can never be picked up on this macOS-only cross branch. + for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-cp${CONDA_PY}-"*macosx*.whl; do [ -e "$w" ] && { code_whl="$w"; break; } done [ -n "$code_whl" ] || { echo "ERROR: no ${PKG_NAME}==${PKG_VERSION} cp${CONDA_PY} wheel in '$WHEELS_DIR'" >&2; exit 1; } diff --git a/conda/mssql-python/meta.yaml b/conda/mssql-python/meta.yaml index cbc39fdd..4ede11dd 100644 --- a/conda/mssql-python/meta.yaml +++ b/conda/mssql-python/meta.yaml @@ -45,6 +45,8 @@ requirements: - python # conda does NOT inherit the wheel's install_requires, so pin azure-identity here. - azure-identity >=1.12.0 + # Defaults' Windows ARM64 Python 3.12 does not export its CPython ABI constraint. + - python_abi 3.12.* *_cp312 # [win and arm64 and py == 312] # ODBC Driver 18 payload deps (the driver ships inside this package, so its # security-serviced deps are declared here). OpenSSL for TLS is dlopen'd, so # overlinking can't see it; pinned <4 (Driver 18 supports the OpenSSL 1.1/3.0 ABI @@ -63,8 +65,8 @@ requirements: # msodbcsql18.dll imports VCRUNTIME140.dll but the vendored vcredist ships only # msvcp140.dll; declare the serviced conda runtime. - vc14_runtime # [win] - # Re-assert the wheel's platform floor (conda drops the wheel tag); never stricter. - - __glibc >=2.28 # [linux] + # The complete payload, including the required bulk-copy core, needs GLIBC_2.34. + - __glibc >=2.34 # [linux] - __osx >=15.0 # [osx] test: diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 66db771d..d6590f6d 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -154,6 +154,9 @@ jobs: sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" -C + # Pool eviction tests observe physical connections from a second session. + sqlcmd -S "localhost" -U "sa" -P "$env:DB_PASSWORD" -b -Q "GRANT VIEW SERVER PERFORMANCE STATE TO [testuser]; EXECUTE AS LOGIN = 'testuser'; SELECT connection_id FROM sys.dm_exec_connections WHERE session_id = @@SPID; REVERT;" -C + if ($LASTEXITCODE -ne 0) { throw "Failed to provision the CI test login's DMV-read permission." } displayName: 'Setup database and user for SQL Server 2022' condition: eq(variables['sqlVersion'], 'SQL2022') env: @@ -220,6 +223,9 @@ jobs: sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" -C + # Pool eviction tests observe physical connections from a second session. + sqlcmd -S "localhost" -U "sa" -P "$env:DB_PASSWORD" -b -Q "GRANT VIEW SERVER PERFORMANCE STATE TO [testuser]; EXECUTE AS LOGIN = 'testuser'; SELECT connection_id FROM sys.dm_exec_connections WHERE session_id = @@SPID; REVERT;" -C + if ($LASTEXITCODE -ne 0) { throw "Failed to provision the CI test login's DMV-read permission." } displayName: 'Setup database and user for SQL Server 2025' condition: eq(variables['sqlVersion'], 'SQL2025') env: diff --git a/eng/requirements-build-macos.txt b/eng/requirements-build-macos.txt index e1e3f6f2..6a1f26f2 100644 --- a/eng/requirements-build-macos.txt +++ b/eng/requirements-build-macos.txt @@ -1240,6 +1240,81 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/../requirements.txt requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed diff --git a/eng/requirements-test-linux.txt b/eng/requirements-test-linux.txt index ef1d0a82..37610921 100644 --- a/eng/requirements-test-linux.txt +++ b/eng/requirements-test-linux.txt @@ -1230,6 +1230,81 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r eng/../requirements.txt requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed diff --git a/eng/scripts/_conda_pkg.py b/eng/scripts/_conda_pkg.py index cfc87cdb..3ed2d23d 100644 --- a/eng/scripts/_conda_pkg.py +++ b/eng/scripts/_conda_pkg.py @@ -14,9 +14,74 @@ import io import json +import re import tarfile import zipfile -from typing import Any, Iterator +from typing import Any, Iterable, Iterator + + +def validate_native_contract( + members: Iterable[tuple[str, bytes]], index: dict[str, Any] +) -> list[str]: + """Require a target binding and importable core filename; platform audits check headers. + + Wheels may include bindings for several Python minors. The core uses Python's + normal extension loader, including its stable-ABI suffix. These static checks + do not replace native import/feature qualification. + """ + pins = [ + d + for d in index.get("depends", []) + if isinstance(d, str) and d.split()[:1] == ["python_abi"] + ] + abi = re.fullmatch(r"python_abi (3\.\d+)\.\* \*_cp(3\d+)", pins[0]) if len(pins) == 1 else None + if abi is None or abi[1].replace(".", "") != abi[2]: + return ["expected a matching normal CPython python_abi pin"] + subdir = index["subdir"] + windows = subdir.startswith("win-") + suffix = "pyd" if windows else "so" + prefix = "Lib" if windows else f"lib/python{abi[1]}" + root = f"{prefix}/site-packages/" + names = [name.replace("\\", "/") for name, _ in members] + bindings = [ + name + for name in names + if re.fullmatch( + rf"{re.escape(root)}mssql_python/ddbc_bindings\.cp{abi[2]}-[^/]+\.{suffix}", name + ) + ] + cores = [ + name + for name in names + if re.fullmatch( + rf"{re.escape(root)}mssql_py_core/mssql_py_core(?:\.[^/]+)?\.{suffix}", name + ) + ] + arch = { + "win-64": "win_amd64", + "win-arm64": "win_arm64", + "linux-64": "x86_64-linux-gnu", + "linux-aarch64": "aarch64-linux-gnu", + "osx-64": "darwin", + "osx-arm64": "darwin", + }[subdir] + core_names = ( + (f"mssql_py_core.cp{abi[2]}-{arch}.pyd", "mssql_py_core.pyd") + if windows + else (f"mssql_py_core.cpython-{abi[2]}-{arch}.so", "mssql_py_core.abi3.so") + ) + errors = [] + if len(bindings) != 1: + errors.append( + f"expected exactly one normal cp{abi[2]} native binding; found {len(bindings)}" + ) + if len(cores) != 1: + errors.append( + f"expected exactly one required mssql_py_core native extension; found {len(cores)}" + ) + elif cores[0].rsplit("/", 1)[-1] not in core_names: + errors.append(f"{cores[0]}: mssql_py_core is incompatible with normal cp{abi[2]} {subdir}") + return errors def zstd_decompress(raw: bytes) -> bytes: diff --git a/eng/scripts/assert_macho_arch.py b/eng/scripts/assert_macho_arch.py index d4afe6ed..77ba03a6 100644 --- a/eng/scripts/assert_macho_arch.py +++ b/eng/scripts/assert_macho_arch.py @@ -28,7 +28,11 @@ import struct import sys -from _conda_pkg import iter_payload_members as _iter_payload_members, read_index +from _conda_pkg import ( + iter_payload_members as _iter_payload_members, + read_index, + validate_native_contract, +) # Mach-O cputype (mach/machine.h): the base type OR'd with the 64-bit ABI flag -> lipo name. _CPU_ARCH_ABI64 = 0x01000000 @@ -152,7 +156,8 @@ def audit_package(path: str) -> list[str]: """Return violation strings for one package (empty == clean / skipped non-macOS).""" base_name = os.path.basename(path) try: - subdir = read_subdir(path) + index = read_index(path) + subdir = str(index.get("subdir", "")) except Exception as exc: # malformed must FAIL, never silently skip return [f"{base_name}: unreadable/malformed package metadata ({exc})."] @@ -166,7 +171,7 @@ def audit_package(path: str) -> list[str]: except ValueError as exc: # malformed payload (e.g. .conda missing pkg-*.tar.zst) return [f"{base_name}: unreadable/malformed package payload ({exc})."] - errors: list[str] = [] + errors = validate_native_contract(members, index) binding_seen = 0 target_driver_libraries: set[str] = set() for name, data in members: @@ -178,6 +183,8 @@ def audit_package(path: str) -> list[str]: if "/mssql_python/" in low and base_low.startswith("ddbc_bindings") and low.endswith(".so"): binding_seen += 1 required_arch = expected + elif "/mssql_py_core/" in low and low.endswith(".so"): + required_arch = expected elif "/mssql_python_odbc/libs/macos/" in low and low.endswith(".dylib"): relative = low.split("/mssql_python_odbc/libs/macos/", 1)[1] parts = relative.split("/") diff --git a/eng/scripts/assert_pe_machine.py b/eng/scripts/assert_pe_machine.py index ae339a5e..05cc0a27 100644 --- a/eng/scripts/assert_pe_machine.py +++ b/eng/scripts/assert_pe_machine.py @@ -24,7 +24,11 @@ import struct import sys -from _conda_pkg import iter_payload_members as _iter_payload_members, read_index +from _conda_pkg import ( + iter_payload_members as _iter_payload_members, + read_index, + validate_native_contract, +) # IMAGE_FILE_MACHINE_* (winnt.h): the PE COFF Machine field -> a short name. _MACHINES = { @@ -86,7 +90,8 @@ def audit_package(path: str) -> list[str]: """Return violation strings for one package (empty == clean / skipped non-Windows).""" base_name = os.path.basename(path) try: - subdir = read_subdir(path) + index = read_index(path) + subdir = str(index.get("subdir", "")) except Exception as exc: # malformed must FAIL, never silently skip return [f"{base_name}: unreadable/malformed package metadata ({exc})."] @@ -101,7 +106,7 @@ def audit_package(path: str) -> list[str]: except ValueError as exc: # malformed payload (e.g. .conda missing pkg-*.tar.zst) return [f"{base_name}: unreadable/malformed package payload ({exc})."] - errors: list[str] = [] + errors = validate_native_contract(members, index) native_seen = 0 binding_seen = 0 driver_dll_seen = 0 diff --git a/eng/scripts/audit_bundled_binaries.py b/eng/scripts/audit_bundled_binaries.py index a4afa81d..f7698e9b 100644 --- a/eng/scripts/audit_bundled_binaries.py +++ b/eng/scripts/audit_bundled_binaries.py @@ -44,17 +44,25 @@ import glob import os import posixpath +import re import struct import sys from typing import Any, Iterable, TypedDict -from _conda_pkg import iter_payload_members as _iter_payload_members, read_index +from _conda_pkg import ( + iter_payload_members as _iter_payload_members, + read_index, + validate_native_contract, +) # --- ELF constants --------------------------------------------------------- _DT_NEEDED = 1 _DT_STRTAB = 5 +_DT_STRSZ = 10 _DT_RPATH = 15 _DT_RUNPATH = 29 +_DT_VERNEED = 0x6FFFFFFE +_DT_VERNEEDNUM = 0x6FFFFFFF _PT_LOAD = 1 _PT_DYNAMIC = 2 @@ -104,6 +112,7 @@ class _ElfDynamicInfo(TypedDict): runpath: str | None rpath: str | None needed: list[str] + glibc_required: list[tuple[int, ...]] def _is_elf(data: bytes) -> bool: @@ -130,9 +139,11 @@ def elf_dynamic(data: bytes) -> _ElfDynamicInfo: table that a stripped/rewritten binary might not carry. Handles ELF32/ELF64 and both endiannesses; the shipped drivers are ELF64-LE. """ - out: _ElfDynamicInfo = {"runpath": None, "rpath": None, "needed": []} + out: _ElfDynamicInfo = {"runpath": None, "rpath": None, "needed": [], "glibc_required": []} if not _is_elf(data): - return out + raise ValueError("not a complete ELF header") + if data[4] not in (1, 2) or data[5] not in (1, 2): + raise ValueError("invalid ELF class or endianness") is64 = data[4] == 2 en = "<" if data[5] == 1 else ">" @@ -144,15 +155,20 @@ def elf_dynamic(data: bytes) -> _ElfDynamicInfo: e_phoff = struct.unpack_from(en + "I", data, 0x1C)[0] e_phentsize = struct.unpack_from(en + "H", data, 0x2A)[0] e_phnum = struct.unpack_from(en + "H", data, 0x2C)[0] - if not e_phoff or not e_phnum: - return out + if ( + not e_phoff + or not e_phnum + or e_phentsize < (56 if is64 else 32) + or e_phoff + e_phnum * e_phentsize > len(data) + ): + raise ValueError("invalid or truncated ELF program headers") loads = [] # (p_vaddr, p_offset, p_filesz) dyn = None # (p_offset, p_filesz) for i in range(e_phnum): off = e_phoff + i * e_phentsize if off + e_phentsize > len(data): - return out + raise ValueError("truncated ELF program header") p_type = struct.unpack_from(en + "I", data, off)[0] if is64: p_offset = struct.unpack_from(en + "Q", data, off + 8)[0] @@ -162,25 +178,33 @@ def elf_dynamic(data: bytes) -> _ElfDynamicInfo: p_offset = struct.unpack_from(en + "I", data, off + 4)[0] p_vaddr = struct.unpack_from(en + "I", data, off + 8)[0] p_filesz = struct.unpack_from(en + "I", data, off + 16)[0] + if p_offset + p_filesz > len(data): + raise ValueError("ELF segment extends beyond the file") if p_type == _PT_LOAD: loads.append((p_vaddr, p_offset, p_filesz)) elif p_type == _PT_DYNAMIC: dyn = (p_offset, p_filesz) if dyn is None: - return out + raise ValueError("ELF has no PT_DYNAMIC segment") dyn_off, dyn_size = dyn - def vaddr_to_off(vaddr: int) -> int | None: + def vaddr_to_off(vaddr: int, size: int = 1) -> int: for v, o, sz in loads: - if v <= vaddr < v + sz: + if v <= vaddr and vaddr + size <= v + sz: return vaddr - v + o - return None + raise ValueError("ELF dynamic address is outside a file-backed PT_LOAD segment") strtab_vaddr = None + strtab_size = None + verneed_vaddr = None + verneed_num = None runpath_rel = None rpath_rel = None needed_rel: list[int] = [] entsize = 16 if is64 else 8 + terminated = False + if dyn_size % entsize: + raise ValueError("ELF dynamic segment has a partial entry") for off in range(dyn_off, dyn_off + dyn_size, entsize): if off + entsize > len(data): break @@ -191,6 +215,7 @@ def vaddr_to_off(vaddr: int) -> int | None: d_tag = struct.unpack_from(en + "i", data, off)[0] d_val = struct.unpack_from(en + "I", data, off + 4)[0] if d_tag == 0: # DT_NULL terminates the array + terminated = True break if d_tag == _DT_STRTAB: strtab_vaddr = d_val @@ -200,25 +225,78 @@ def vaddr_to_off(vaddr: int) -> int | None: rpath_rel = d_val elif d_tag == _DT_NEEDED: needed_rel.append(d_val) - if strtab_vaddr is None: - return out - strtab_off = vaddr_to_off(strtab_vaddr) - if strtab_off is None: - return out + elif d_tag == _DT_STRSZ: + strtab_size = d_val + elif d_tag == _DT_VERNEED: + verneed_vaddr = d_val + elif d_tag == _DT_VERNEEDNUM: + verneed_num = d_val + if not terminated or strtab_vaddr is None or not strtab_size: + raise ValueError("ELF dynamic segment lacks DT_NULL, DT_STRTAB or DT_STRSZ") + strtab_off = vaddr_to_off(strtab_vaddr, strtab_size) def read_str(rel: int) -> str: + if not 0 <= rel < strtab_size: + raise ValueError("ELF string offset is outside DT_STRTAB") pos = strtab_off + rel - end = data.find(b"\x00", pos) - return data[pos : (end if end >= 0 else len(data))].decode("utf-8", "replace") + end = data.find(b"\x00", pos, strtab_off + strtab_size) + if end < 0: + raise ValueError("unterminated ELF dynamic string") + return data[pos:end].decode("utf-8", "strict") if runpath_rel is not None: out["runpath"] = read_str(runpath_rel) if rpath_rel is not None: out["rpath"] = read_str(rpath_rel) out["needed"] = [read_str(n) for n in needed_rel] + if (verneed_vaddr is None) != (verneed_num is None): + raise ValueError("ELF version requirements need both DT_VERNEED and DT_VERNEEDNUM") + if verneed_vaddr is not None: + if not verneed_num or verneed_num > len(data) // 16: + raise ValueError("invalid ELF version requirement count") + current = verneed_vaddr + for number in range(verneed_num): + offset = vaddr_to_off(current, 16) + version, count, library, aux, next_need = struct.unpack_from(en + "HHIII", data, offset) + if version != 1 or not count or count > len(data) // 16 or aux < 16: + raise ValueError("invalid ELF version requirement record") + read_str(library) + auxiliary = current + aux + for item in range(count): + offset = vaddr_to_off(auxiliary, 16) + _, _, _, name, next_aux = struct.unpack_from(en + "IHHII", data, offset) + requirement = read_str(name) + if requirement.startswith("GLIBC_"): + match = re.fullmatch(r"GLIBC_(\d+(?:\.\d+)+)", requirement) + if match is None: + raise ValueError(f"unsupported glibc symbol requirement {requirement}") + out["glibc_required"].append(tuple(map(int, match[1].split(".")))) + if item < count - 1 and next_aux < 16: + raise ValueError("truncated ELF version auxiliary chain") + if item == count - 1 and next_aux != 0: + raise ValueError("ELF version auxiliary count disagrees with chain") + auxiliary += next_aux + if number < verneed_num - 1 and next_need < 16: + raise ValueError("truncated ELF version requirement chain") + if number == verneed_num - 1 and next_need != 0: + raise ValueError("ELF version requirement count disagrees with chain") + current += next_need return out +def declared_glibc_floor(index: dict[str, Any]) -> tuple[int, ...]: + """Require a single explicit minimum; wheel platform tags are not symbol-floor evidence.""" + specs = [ + d for d in index.get("depends", []) if isinstance(d, str) and d.split()[:1] == ["__glibc"] + ] + if len(specs) != 1: + raise ValueError("expected exactly one __glibc >=VERSION dependency") + match = re.fullmatch(r"__glibc\s+>=(\d+(?:\.\d+)+)", specs[0]) + if match is None: + raise ValueError(f"unsupported __glibc dependency: {specs[0]!r}") + return tuple(map(int, match[1].split("."))) + + def effective_runpath(dyn: _ElfDynamicInfo) -> str | None: """The loader ignores ``DT_RPATH`` when ``DT_RUNPATH`` is present.""" return dyn["runpath"] if dyn["runpath"] is not None else dyn["rpath"] @@ -342,6 +420,13 @@ def audit_package(path: str) -> list[str]: except ValueError as exc: # malformed payload (e.g. .conda missing pkg-*.tar.zst) return [f"{base_name}: unreadable/malformed package payload ({exc})."] + errors.extend(validate_native_contract(members, index)) + try: + glibc_floor = declared_glibc_floor(index) + except (TypeError, ValueError) as exc: + errors.append(f"{base_name}: invalid glibc compatibility metadata: {exc}") + glibc_floor = None + for name, data in members: base = posixpath.basename(name) norm = "/" + name @@ -364,11 +449,15 @@ def audit_package(path: str) -> list[str]: is_driver = any(base.startswith(p) for p in _DRIVER_PREFIXES) is_inst = base == _ODBCINST - if not (is_driver or is_inst): + is_native = data.startswith(b"\x7fELF") or base.endswith(".so") or ".so." in base + if not (is_native or is_driver or is_inst): continue if not _is_elf(data): errors.append(f"{name}: expected an ELF binary but the header is not ELF.") continue + if data[4:6] != b"\x02\x01": + errors.append(f"{name}: expected an ELF64 little-endian binary for '{subdir}'.") + continue # Architecture gate: the ELF machine MUST match the package's conda subdir, so # an x86_64 driver mislabeled under a linux-aarch64 package (which the emulated @@ -379,10 +468,26 @@ def audit_package(path: str) -> list[str]: errors.append( f"{name}: ELF machine {mach} ({machine_name}) does " f"not match the '{subdir}' package arch {expected_machine} " - f"({_MACHINE_NAME[expected_machine]}) -- wrong-arch/mislabeled driver." + f"({_MACHINE_NAME[expected_machine]}) -- wrong-arch/mislabeled native binary." ) - dyn = elf_dynamic(data) + try: + dyn = elf_dynamic(data) + except (ValueError, struct.error) as exc: + errors.append(f"{name}: invalid ELF dynamic metadata ({exc}).") + continue + for required in dyn["glibc_required"]: + if glibc_floor is not None: + width = max(len(required), len(glibc_floor)) + if required + (0,) * (width - len(required)) > glibc_floor + (0,) * ( + width - len(glibc_floor) + ): + errors.append( + f"{name}: requires GLIBC_{'.'.join(map(str, required))} but archive " + f"declares __glibc >={'.'.join(map(str, glibc_floor))}." + ) + if not (is_driver or is_inst): + continue raw_runpath = effective_runpath(dyn) entries = _entries(raw_runpath) needed = dyn["needed"] diff --git a/requirements.txt b/requirements.txt index b028797a..2164be12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Testing dependencies pytest pytest-cov +PyYAML zstandard coverage unittest-xml-reporting diff --git a/tests/test_029_bundled_binary_audit.py b/tests/test_029_bundled_binary_audit.py index b516cc0c..26f18f39 100644 --- a/tests/test_029_bundled_binary_audit.py +++ b/tests/test_029_bundled_binary_audit.py @@ -53,7 +53,7 @@ def _load_module(): audit = _load_module() -def _make_elf64(runpath=None, rpath=None, needed=(), machine=62): +def _make_elf64(runpath=None, rpath=None, needed=(), machine=62, versions=()): """Build a minimal, self-consistent ELF64-LE with real program headers. Emits a PT_LOAD (vaddr == file offset, covering the whole file) + a PT_DYNAMIC, @@ -78,8 +78,17 @@ def add_str(s): rp_rel = add_str(runpath) if runpath is not None else None rpath_rel = add_str(rpath) if rpath is not None else None need_rels = [add_str(n) for n in needed] - - dynamic_off = dynstr_off + len(dynstr) + version_rels = [add_str(n) for n in versions] + library_rel = add_str("libc.so.6") if versions else 0 + verneed_off = dynstr_off + len(dynstr) + verneed = b"" + if versions: + verneed = struct.pack("=3,<4"] +_GOOD_DEPENDS = [ + "python >=3.12,<3.13.0a0", + "python_abi 3.12.* *_cp312", + "__glibc >=2.34", + "azure-identity", + "krb5", + "libtool", + "openssl >=3,<4", +] +_BINDING = "lib/python3.12/site-packages/mssql_python/ddbc_bindings.cp312-x86_64.so" +_CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-x86_64-linux-gnu.so" _DISTROS_BY_SUBDIR = { "linux-64": ("alpine", "debian_ubuntu", "rhel", "suse"), "linux-aarch64": ("alpine", "debian_ubuntu", "rhel"), @@ -151,6 +174,7 @@ def _make_pkg( inst_needed=None, machine=62, distros=None, + native_payload=None, ): """Write a minimal package with complete per-distro driver trees by default.""" p = tmp_path / "mssql-python-1.13.0-py312_0.tar.bz2" @@ -174,6 +198,16 @@ def add(name, data): ).encode(), ) arch = "arm64" if subdir == "linux-aarch64" else "x86_64" + extension_arch = "aarch64" if subdir == "linux-aarch64" else "x86_64" + if native_payload is None: + native_payload = { + _BINDING.replace("x86_64", extension_arch): _make_elf64(machine=machine), + _CORE.replace("x86_64", extension_arch): _make_elf64( + machine=machine, versions=("GLIBC_2.2.5", "GLIBC_2.34") + ), + } + for name, data in native_payload.items(): + add(name, data) selected_distros = distros or _DISTROS_BY_SUBDIR.get(subdir, ("debian_ubuntu",)) for distro in selected_distros: libdir = ( @@ -234,6 +268,115 @@ def test_audit_passes_with_exact_climb(tmp_path): assert audit.audit_package(_make_pkg(tmp_path)) == [] +@pytest.mark.parametrize("missing", [_BINDING, _CORE]) +def test_full_feature_package_requires_binding_and_core(tmp_path, missing): + native = {_BINDING: _make_elf64(), _CORE: _make_elf64()} + del native[missing] + errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + assert any("exactly one" in error for error in errors) + + +@pytest.mark.parametrize("component", [_BINDING, _CORE]) +def test_every_required_extension_checks_elf_arch_not_filename(tmp_path, component): + native = {_BINDING: _make_elf64(), _CORE: _make_elf64()} + native[component] = _make_elf64(machine=183) + errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + assert any(component in error and "does not match" in error for error in errors) + + +@pytest.mark.parametrize("component", [_BINDING, _CORE]) +@pytest.mark.parametrize("wrong_tag", ["311", "312t", "312d"]) +def test_required_extensions_reject_wrong_or_non_normal_abi(tmp_path, component, wrong_tag): + native = {_BINDING: _make_elf64(), _CORE: _make_elf64()} + data = native.pop(component) + native[component.replace("312", wrong_tag)] = data + assert audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + + +def test_core_abi3_and_extra_normal_bindings_are_supported(tmp_path): + native = { + _BINDING: _make_elf64(), + _BINDING.replace("312", "310"): _make_elf64(), + _CORE.replace("cpython-312-x86_64-linux-gnu", "abi3"): _make_elf64(), + } + assert audit.audit_package(_make_pkg(tmp_path, native_payload=native)) == [] + + +@pytest.mark.parametrize( + "abi", + [ + "python_abi 3.12.* *_cp312t", + "python_abi 3.12.* *_cp311", + "python_abi 3.11.* *_cp311", + "python_abi >=3.12", + ], +) +def test_archive_requires_consistent_normal_python_abi_metadata(tmp_path, abi): + depends = [abi if d.startswith("python_abi ") else d for d in _GOOD_DEPENDS] + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("python_abi" in error or "native binding" in error for error in errors) + + +@pytest.mark.parametrize("declared", ["2.28", "2.33"]) +def test_core_symbol_floor_cannot_exceed_archive_glibc_minimum(tmp_path, declared): + depends = [f"__glibc >={declared}" if d.startswith("__glibc ") else d for d in _GOOD_DEPENDS] + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any(_CORE in error and "GLIBC_2.34" in error for error in errors) + + +@pytest.mark.parametrize("declared", ["2.34", "2.34.0", "2.35"]) +def test_core_symbol_floor_compatible_with_archive_minimum(tmp_path, declared): + depends = [f"__glibc >={declared}" if d.startswith("__glibc ") else d for d in _GOOD_DEPENDS] + assert audit.audit_package(_make_pkg(tmp_path, depends=depends)) == [] + + +def test_symbol_floor_is_read_from_version_needs_not_arbitrary_bytes(tmp_path): + native = { + _BINDING: _make_elf64(), + _CORE: _make_elf64(versions=("GLIBC_2.34",)) + b"GLIBC_99.99\x00", + } + assert audit.audit_package(_make_pkg(tmp_path, native_payload=native)) == [] + + +def test_auxiliary_native_library_symbol_floor_is_checked(tmp_path): + extra = "lib/python3.12/site-packages/mssql_py_core.libs/libsupport.so.1" + native = { + _BINDING: _make_elf64(), + _CORE: _make_elf64(), + extra: _make_elf64(versions=("GLIBC_2.35",)), + } + errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + assert any(extra in error and "GLIBC_2.35" in error for error in errors) + + +@pytest.mark.parametrize("spec", [None, "__glibc >=2.28|>=2.34", "__glibc >=2.34junk"]) +def test_missing_or_ambiguous_glibc_floor_is_rejected(tmp_path, spec): + depends = [d for d in _GOOD_DEPENDS if not d.startswith("__glibc")] + if spec: + depends.append(spec) + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("glibc compatibility metadata" in error for error in errors) + + +@pytest.mark.parametrize("damage", ["truncated", "version-address", "aux-chain", "class"]) +def test_malformed_required_core_elf_cannot_pass(tmp_path, damage): + core = bytearray(_make_elf64(versions=("GLIBC_2.2.5", "GLIBC_2.34"))) + if damage == "truncated": + core = core[:70] + elif damage == "version-address": + tag = core.index(struct.pack("=3.12,<3.13.0a0", "azure-identity >=1.12.0"] + if state != "missing-abi": + depends.append(pin if state != "wrong-abi" else "python_abi 3.12.* *_cp313") + core = "Lib/site-packages/mssql_py_core/mssql_py_core.cp312-win_arm64.pyd" + payload = { + "Lib/site-packages/mssql_python/ddbc_bindings.cp312-arm64.pyd": _fake_pe(_ARM64), + core: _fake_pe(_ARM64), + "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/msodbcsql18.dll": _fake_pe(_ARM64), + "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/mssql-auth.dll": _fake_pe(_ARM64), + } + if state == "missing": + del payload[core] + elif state == "wrong-arch": + payload[core] = _fake_pe(_AMD64) + elif state == "wrong-tag": + payload[core.replace("312", "311")] = payload.pop(core) + elif state == "abi3": + payload[core.replace(".cp312-win_arm64", "")] = payload.pop(core) + errors = ape.audit_package(_make_conda(tmp_path, "win-arm64", payload, depends=depends)) + if state in ("valid", "abi3"): + assert errors == [] + elif state in ("missing-abi", "wrong-abi"): + assert any("matching normal CPython python_abi pin" in error for error in errors) + else: + assert any("mssql_py_core" in error for error in errors) + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") def test_win_arm64_driver_dlls_in_x64_directory_fail_presence_gate(tmp_path): # Correct ARM64 machine fields do not help when the loader searches windows/arm64, diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index 413be486..be78c15b 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -338,7 +338,7 @@ def _fake_run(cmd, *args, **kwargs): import_calls = [ (cmd, cwd) for cmd, cwd in calls - if "-c" in cmd and any("mssql_python" in str(a) for a in cmd) + if "-c" in cmd and any("mssql_python" in str(a) or "mssql_py_core" in str(a) for a in cmd) ] assert import_calls, "verify() never issued an `import mssql_python` probe" for cmd, cwd in import_calls: @@ -346,6 +346,10 @@ def _fake_run(cmd, *args, **kwargs): f"import probe ran from {cwd!r}, not the neutral workdir {str(workdir)!r} -- the " f"repo source tree would shadow the conda-installed package" ) + codes = [cmd[-1] for cmd, _ in import_calls] + assert codes[0] == mod._core_probe() + assert codes.count(mod._core_probe()) == 1 + assert "import mssql_python" not in codes[0] def test_verify_restores_cwd_when_the_phase_fails(tmp_path, monkeypatch): @@ -599,6 +603,7 @@ def test_gather_wheels_rejects_multiple_odbc_matches(tmp_path): ("target_subdir", "expected_channels"), [ ("", ["microsoft", "conda-forge"]), + ("win-64", ["microsoft", "conda-forge"]), ("osx-arm64", ["microsoft", "conda-forge"]), ("linux-aarch64", ["microsoft", "conda-forge"]), ("win-arm64", ["defaults", "microsoft", "conda-forge"]), @@ -636,3 +641,103 @@ def _capture_run(command, **_kwargs): assert [command[index + 1] for index, arg in enumerate(command) if arg == "-c"] == ( expected_channels ) + + +@pytest.mark.parametrize("state", ["native", "pure-python", "foreign"]) +def test_core_probe_requires_native_extension_from_installed_prefix(state, tmp_path, monkeypatch): + mod = _load_orchestrator() + prefix = tmp_path / "prefix" + monkeypatch.setattr(sys, "prefix", str(prefix)) + package = types.ModuleType("mssql_py_core") + package.__file__ = str(prefix / "mssql_py_core" / "__init__.py") + monkeypatch.setitem(sys.modules, "mssql_py_core", package) + if state != "pure-python": + native = types.ModuleType("mssql_py_core.mssql_py_core") + native.__file__ = str((tmp_path / "foreign" if state == "foreign" else prefix) / "core.pyd") + native.__loader__ = importlib.machinery.ExtensionFileLoader( + native.__name__, native.__file__ + ) + monkeypatch.setitem(sys.modules, native.__name__, native) + if state == "native": + exec(mod._core_probe(), {}) + else: + with pytest.raises(AssertionError, match="native extension|outside installed prefix"): + exec(mod._core_probe(), {}) + + +def test_core_failure_blocks_api_preload(tmp_path, monkeypatch): + mod = _load_orchestrator() + calls = [] + + def fake_run(cmd, **kwargs): + calls.append(list(cmd)) + return types.SimpleNamespace(returncode=17 if mod._core_probe() in cmd else 0, stdout="") + + monkeypatch.setattr( + mod, + "subprocess", + types.SimpleNamespace(run=fake_run, PIPE=subprocess.PIPE, STDOUT=subprocess.STDOUT), + ) + with pytest.raises(SystemExit): + mod._verify_impl( + "conda", "channel", str(tmp_path), ["3.12"], "1.14.0", "linux-64", False, {} + ) + assert any(mod._core_probe() in cmd for cmd in calls) + assert not any("BINDING_OK" in str(cmd) for cmd in calls) + + +@pytest.mark.parametrize("subdir", ["win-64", "win-arm64"]) +def test_both_windows_targets_run_native_audit(subdir, monkeypatch): + mod = _load_orchestrator() + calls = [] + monkeypatch.setattr(mod, "run", lambda cmd, **kwargs: calls.append(cmd)) + mod.audit_packages( + "conda", "builder", str(_ORCH_PATH.parents[2] / "conda"), "output", subdir, {} + ) + pe_calls = [ + cmd for cmd in calls if any(str(arg).endswith("assert_pe_machine.py") for arg in cmd) + ] + assert len(pe_calls) == 1 + assert pe_calls[0][-2:] == ["--subdir", subdir] + + +def test_build_does_not_automatically_accept_channel_terms(monkeypatch): + mod = _load_orchestrator() + monkeypatch.delenv("CONDA_PLUGINS_AUTO_ACCEPT_TOS", raising=False) + assert "CONDA_PLUGINS_AUTO_ACCEPT_TOS" not in mod.build_env( + "1.14.0", "18.6.2.1", "wheels", "win-arm64" + ) + + +@pytest.mark.parametrize("subdir", ["win-64", "win-arm64"]) +def test_main_routes_effective_target_to_native_audit(subdir, tmp_path, monkeypatch): + mod = _load_orchestrator() + targets = [] + monkeypatch.setattr(mod, "gather_wheels", lambda *args: ("1.14.0", "18.6.2.1")) + monkeypatch.setattr(mod, "find_or_install_conda", lambda *args: "conda") + monkeypatch.setattr(mod, "create_builder_env", lambda *args: "builder") + monkeypatch.setattr(mod, "detect_pythons", lambda *args: ["3.12"]) + monkeypatch.setattr(mod, "make_verify_channel", lambda *args: "channel") + for name in ("run", "build_packages", "verify", "stage"): + monkeypatch.setattr(mod, name, lambda *args, **kwargs: None) + monkeypatch.setattr(mod, "audit_packages", lambda *args: targets.append(args[-2])) + args = [ + "--mssql-wheel-dir", + str(tmp_path / "wheels"), + "--odbc-wheel-dir", + str(tmp_path / "wheels"), + "--odbc-wheel-filter", + "*.whl", + "--recipe-root", + str(_ORCH_PATH.parents[2] / "conda"), + "--output-dir", + str(tmp_path / "out"), + "--stage-dir", + str(tmp_path / "stage"), + "--conda-subdir", + subdir, + ] + if subdir == "win-arm64": + args += ["--conda-target-subdir", subdir] + assert mod.main(args) == 0 + assert targets == [subdir] diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index 2eda5058..8532a0cb 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -145,6 +145,7 @@ def _make_conda(tmp_path, subdir, payload): tf.addfile(ti, io.BytesIO(data)) index = {"name": "mssql-python", "version": "1.13.0", "build": "py312_0", "subdir": subdir} + index["depends"] = ["python_abi 3.12.* *_cp312"] idx = json.dumps(index).encode() info_buf = io.BytesIO() with tarfile.open(fileobj=info_buf, mode="w") as tf: @@ -160,6 +161,7 @@ def _make_conda(tmp_path, subdir, payload): _BINDING = "lib/python3.12/site-packages/mssql_python/ddbc_bindings.cp312-darwin.so" +_CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-darwin.so" _DRIVER_ROOT = "lib/python3.12/site-packages/mssql_python_odbc/libs/macos" _DRIVER_LIBRARIES = ( "libltdl.7.dylib", @@ -173,7 +175,7 @@ def _realistic_payload(binding, arm64=None, x86_64=None): """Mirror the wheel's two architecture-specific four-library driver directories.""" arm64 = arm64 or _fake_macho_thin(_ARM64) x86_64 = x86_64 or _fake_macho_thin(_X86_64) - payload = {_BINDING: binding} + payload = {_BINDING: binding, _CORE: _fake_macho_fat([_X86_64, _ARM64])} for library in _DRIVER_LIBRARIES: payload[f"{_DRIVER_ROOT}/arm64/lib/{library}"] = arm64 payload[f"{_DRIVER_ROOT}/x86_64/lib/{library}"] = x86_64 @@ -192,6 +194,24 @@ def test_osx_packages_accept_real_split_driver_layout(tmp_path, subdir): assert mac.audit_package(p) == [] +@pytest.mark.parametrize("state", ["missing", "wrong-arch", "wrong-tag", "abi3"]) +def test_required_core_contract(tmp_path, state): + payload = _realistic_payload(_fake_macho_fat([_X86_64, _ARM64])) + if state == "missing": + del payload[_CORE] + elif state == "wrong-arch": + payload[_CORE] = _fake_macho_thin(_X86_64) + elif state == "wrong-tag": + payload[_CORE.replace("312", "311")] = payload.pop(_CORE) + else: + payload[_CORE.replace("cpython-312-darwin", "abi3")] = payload.pop(_CORE) + errors = mac.audit_package(_make_conda(tmp_path, "osx-arm64", payload)) + if state == "abi3": + assert errors == [] + else: + assert any("mssql_py_core" in error for error in errors) + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") @pytest.mark.parametrize("missing_library", _DRIVER_LIBRARIES) def test_target_driver_runtime_requires_every_library(tmp_path, missing_library): From d28fef696d5c8ffc059ec1d3388e3e0ae2eed8c3 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:38:41 +0530 Subject: [PATCH 02/12] CHORE: Remove redundant pipeline-only tests Delete the release dependency workflow and lockfile text test module. Remove YAML-only Conda cases and a recipe wording assertion while preserving native audit, ABI, core-load, subprocess error handling, and security coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_030_pe_machine_assert.py | 2 - tests/test_034_conda_verify_cwd.py | 54 --- tests/test_release_build_dependencies.py | 397 ----------------------- 3 files changed, 453 deletions(-) delete mode 100644 tests/test_release_build_dependencies.py diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index b592762d..725109ea 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -185,8 +185,6 @@ def test_win_arm64_arm64_binaries_pass(tmp_path): ) def test_required_core_contract(tmp_path, state): pin = "python_abi 3.12.* *_cp312" - recipe = _MODULE_PATH.parents[2] / "conda" / "mssql-python" / "meta.yaml" - assert f"- {pin} # [win and arm64 and py == 312]" in recipe.read_text() # The observed defaults CP312 host supplied only the Python range, not an ABI export. depends = ["vc14_runtime", "python >=3.12,<3.13.0a0", "azure-identity >=1.12.0"] if state != "missing-abi": diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index be78c15b..9e7570c1 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -29,8 +29,6 @@ / "scripts" / "build_conda_packages.py" ) -_PIPELINE_PATH = _ORCH_PATH.parent.parent / "conda-build-pipeline.yml" -_CONSOLIDATE_JOB_PATH = _ORCH_PATH.parent.parent / "jobs" / "consolidate-conda-artifacts-job.yml" pytestmark = pytest.mark.skipif( not _ORCH_PATH.exists(), reason=f"orchestrator not present ({_ORCH_PATH})" @@ -45,58 +43,6 @@ def _load_orchestrator(): return mod -def test_best_effort_consolidation_runs_after_upstream_failure(): - pipeline = _PIPELINE_PATH.read_text(encoding="utf-8") - for stage_name in ("CondaWin64", "CondaMacOS", "CondaLinux"): - producer = pipeline.split(f"- stage: {stage_name}", 1)[1] - assert "dependsOn: ValidateWheelProvenance" in producer.split("jobs:", 1)[0] - - stage = pipeline.split("- stage: ConsolidateConda", 1)[1] - dependencies = stage.split("jobs:", 1)[0] - for stage_name in ("CondaWin64", "CondaMacOS", "CondaLinux"): - assert f"- {stage_name}" in dependencies - assert "condition: succeededOrFailed()" in stage.split("jobs:", 1)[0] - - mac_stage = pipeline.split("- stage: CondaMacOS", 1)[1].split("- stage: CondaLinux", 1)[0] - mac_publish = mac_stage.split("displayName: 'Publish macOS conda artifact'", 1)[1] - assert "condition: succeededOrFailed()" in mac_publish.split("inputs:", 1)[0] - - job = _CONSOLIDATE_JOB_PATH.read_text(encoding="utf-8") - consolidate = job.split("- job: ConsolidateArtifacts", 1)[1] - assert "condition: succeededOrFailed()" in consolidate.split("pool:", 1)[0] - - -def test_official_builds_require_main_wheel_provenance(): - pipeline = _PIPELINE_PATH.read_text(encoding="utf-8") - resource = pipeline.split("- pipeline: buildPipeline", 1)[1].split("extends:", 1)[0] - assert "branch: main" in resource - - gate = pipeline.split("- stage: ValidateWheelProvenance", 1)[1].split("- stage: CondaWin64", 1)[ - 0 - ] - assert '[[ -z "${WHEEL_SOURCE_BRANCH:-}" ]]' in gate - assert 'case "$ONEBRANCH_TYPE" in' in gate - assert "Official)" in gate - assert "NonOfficial) ;;" in gate - assert '[[ "$WHEEL_SOURCE_BRANCH" != "refs/heads/main" ]]' in gate - assert "unknown OneBranch type" in gate - assert "ONEBRANCH_TYPE: ${{ variables.effectiveOneBranchType }}" in gate - assert "WHEEL_SOURCE_BRANCH: $(resources.pipeline.buildPipeline.sourceBranch)" in gate - - for stage_name in ("CondaWin64", "CondaMacOS", "CondaLinux"): - producer = pipeline.split(f"- stage: {stage_name}", 1)[1] - assert "dependsOn: ValidateWheelProvenance" in producer.split("jobs:", 1)[0] - - -def test_windows_pool_demand_is_indented_under_demands_key(): - pipeline = _PIPELINE_PATH.read_text(encoding="utf-8") - windows_stage = pipeline.split("- stage: CondaWin64", 1)[1].split("- stage: CondaMacOS", 1)[0] - assert ( - " demands:\n" - " - imageOverride -equals PYTHON-1ES-MMS2022\n" in windows_stage - ) - - @pytest.mark.parametrize( ("target_subdir", "cross_build", "host", "expected"), [ diff --git a/tests/test_release_build_dependencies.py b/tests/test_release_build_dependencies.py deleted file mode 100644 index a66bda64..00000000 --- a/tests/test_release_build_dependencies.py +++ /dev/null @@ -1,397 +0,0 @@ -import re -from pathlib import Path - -import pytest - -ROOT = Path(__file__).parents[1] -ENG = ROOT / "eng" -WORKFLOW = ROOT / ".github" / "workflows" / "refresh-build-dependencies.yml" - -if not ENG.is_dir() or not (ROOT / "OneBranchPipelines").is_dir(): - pytest.skip( - "release dependency contracts require a complete source checkout", - allow_module_level=True, - ) - -DIRECT_REQUIREMENTS = { - "requirements-build-linux": {"pip", "pybind11", "pytest", "setuptools", "wheel"}, - "requirements-build-macos": {"cmake", "cryptography", "pip", "wheel"}, - "requirements-build-odbc": {"build", "pip", "setuptools", "twine", "wheel"}, - "requirements-build-windows": { - "pip", - "psutil", - "pybind11", - "pyodbc", - "pytest", - "setuptools", - "wheel", - "zstandard", - }, -} - -PIPELINE_LOCKS = { - "OneBranchPipelines/stages/build-linux-single-stage.yml": { - "/workspace/eng/requirements-build-linux.txt": 2, - "/workspace/eng/requirements-test-linux.txt": 2, - }, - "OneBranchPipelines/stages/build-macos-single-stage.yml": { - "eng/requirements-build-macos.txt": 1, - }, - "OneBranchPipelines/stages/build-odbc-all-stage.yml": { - "eng/requirements-build-odbc.txt": 1, - }, - "OneBranchPipelines/stages/build-windows-single-stage.yml": { - "eng/requirements-build-windows.txt": 1, - }, -} - -SUPPORTED_PYTHONS = "3.10 3.11 3.12 3.13 3.14" -LINUX_PLATFORMS = ( - "x86_64-manylinux_2_28 aarch64-manylinux_2_28 " - "x86_64-unknown-linux-musl aarch64-unknown-linux-musl" -) - -VALIDATION_MATRIX = { - "Linux build": ( - "linux", - "requirements-build-linux.txt", - SUPPORTED_PYTHONS, - LINUX_PLATFORMS, - ), - "Linux test": ( - "linux", - "requirements-test-linux.txt", - SUPPORTED_PYTHONS, - LINUX_PLATFORMS, - ), - "macOS": ( - "macos", - "requirements-build-macos.txt", - SUPPORTED_PYTHONS, - "x86_64-apple-darwin aarch64-apple-darwin", - ), - "Windows build host": ( - "windows", - "requirements-build-windows.txt", - SUPPORTED_PYTHONS, - "x86_64-pc-windows-msvc", - ), - "ODBC": ( - "odbc", - "requirements-build-odbc.txt", - "3.12", - "x86_64-pc-windows-msvc", - ), -} - -PIN = re.compile( - r"(?P[A-Za-z0-9][A-Za-z0-9._-]*)==(?P[^;\s\\]+)" r"(?:\s*;\s*[^\\]+)?\s+\\" -) -HASH = re.compile(r"--hash=sha256:[0-9a-f]{64}(?:\s+\\)?") -ACTION = re.compile(r"^\s*-\s+uses:\s+([^@\s]+)@([^\s#]+)", re.MULTILINE) - - -def _canonicalize(name): - return re.sub(r"[-_.]+", "-", name).lower() - - -def _active_lines(path): - return [ - line.strip() - for line in path.read_text(encoding="utf-8").splitlines() - if line.strip() and not line.lstrip().startswith("#") - ] - - -def _requirement_names(lines): - return { - _canonicalize(re.match(r"[A-Za-z0-9][A-Za-z0-9._-]*", line).group()) - for line in lines - if not line.startswith(("-r ", "-c ")) - } - - -def _lock_versions(path): - text = path.read_text(encoding="utf-8") - assert "--index-url" not in text - assert "--trusted-host" not in text - - versions = {} - current_name = None - hashes = [] - continued = False - - def finish_entry(): - if current_name is None: - return - assert hashes - assert len(hashes) == len(set(hashes)) - assert not continued - - for line in text.splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - - if line[0].isspace(): - assert current_name is not None and continued - match = HASH.fullmatch(stripped) - assert match - hashes.append(stripped.removesuffix("\\").rstrip()) - continued = stripped.endswith("\\") - continue - - finish_entry() - match = PIN.fullmatch(line) - assert match - current_name = _canonicalize(match["name"]) - assert current_name not in versions - versions[current_name] = match["version"] - hashes = [] - continued = True - - finish_entry() - assert versions - return versions - - -def _section(text, start, end=None): - start_index = text.index(start) - end_index = text.index(end, start_index) if end else len(text) - return text[start_index:end_index] - - -def _matrix_item(section, name): - marker = f" - name: {name}\n" - return _section(section, marker).split("\n - name:", maxsplit=1)[0] - - -@pytest.fixture(scope="module") -def workflow(): - return WORKFLOW.read_text(encoding="utf-8") - - -@pytest.mark.parametrize("stem, expected", DIRECT_REQUIREMENTS.items()) -def test_platform_locks_cover_their_direct_requirements(stem, expected): - assert _requirement_names(_active_lines(ENG / f"{stem}.in")) == expected - assert expected <= _lock_versions(ENG / f"{stem}.txt").keys() - - -@pytest.mark.parametrize( - "lock_name", - [f"{stem}.txt" for stem in DIRECT_REQUIREMENTS] + ["requirements-test-linux.txt"], -) -def test_lockfiles_are_exactly_pinned_and_sha256_hashed(lock_name): - _lock_versions(ENG / lock_name) - - -@pytest.mark.parametrize( - "invalid_lock", - ( - "demo>=1\n", - "demo @ https://example.invalid/demo.whl\n", - f" --hash=sha256:{'a' * 64}\n", - f"demo==1 \\\n --hash=sha512:{'a' * 128}\n", - "demo==1 \\\nother requirement\n", - "demo==1\n", - f"demo==1\n --hash=sha256:{'a' * 64}\n", - f"demo==1 \\\n --hash=sha256:{'a' * 64} \\\n", - f"demo==1 \\\n --hash=sha256:{'a' * 64} junk\n", - f"demo==1 \\\n --hash=sha256:{'a' * 64} \\\n" f" --hash=sha256:{'a' * 64}\n", - ), -) -def test_lockfile_validation_rejects_unpinned_or_malformed_records(tmp_path, invalid_lock): - lock = tmp_path / "invalid-lock.txt" - lock.write_text(invalid_lock, encoding="utf-8") - - with pytest.raises(AssertionError): - _lock_versions(lock) - - -def test_runtime_and_build_requirements_flow_into_combined_locks(): - runtime = _requirement_names(_active_lines(ROOT / "requirements.txt")) - linux_build = _lock_versions(ENG / "requirements-build-linux.txt").keys() - linux_test = _lock_versions(ENG / "requirements-test-linux.txt").keys() - macos = _lock_versions(ENG / "requirements-build-macos.txt").keys() - - assert _active_lines(ENG / "requirements-test-linux.in") == [ - "-r requirements-build-linux.in", - "-c requirements-build-linux.txt", - "-r ../requirements.txt", - ] - assert linux_build <= linux_test - assert runtime <= linux_test - assert runtime <= macos - - -def test_macos_lock_preserves_python_310_cryptography_compatibility(): - assert "cryptography<49" in _active_lines(ENG / "requirements-build-macos.in") - version = _lock_versions(ENG / "requirements-build-macos.txt")["cryptography"] - assert int(version.split(".", maxsplit=1)[0]) < 49 - - -@pytest.mark.parametrize("pipeline_path, locks", PIPELINE_LOCKS.items()) -def test_release_pipelines_only_use_locked_requirements(pipeline_path, locks): - pipeline = (ROOT / pipeline_path).read_text(encoding="utf-8") - installs = [ - line.strip() - for line in pipeline.splitlines() - if "pip install" in line and not line.lstrip().startswith("#") - ] - - for lock, expected_count in locks.items(): - matching = [line for line in installs if f"-r {lock}" in line] - assert len(matching) == expected_count - assert all("--require-hashes" in line for line in matching) - - for command in installs: - if " -r " not in command: - assert '"$WHEEL"' in command or "--no-index --find-links" in command - - -def test_linux_runtime_lock_is_installed_before_each_product_wheel(): - pipeline = (ROOT / "OneBranchPipelines/stages/build-linux-single-stage.yml").read_text( - encoding="utf-8" - ) - runtime_install = ( - "$PY -m pip install -q --require-hashes " "-r /workspace/eng/requirements-test-linux.txt;" - ) - wheel_install = '$PY -m pip install -q "$WHEEL";' - runtime_positions = [ - match.start() for match in re.finditer(re.escape(runtime_install), pipeline) - ] - wheel_positions = [match.start() for match in re.finditer(re.escape(wheel_install), pipeline)] - - assert len(runtime_positions) == len(wheel_positions) == 2 - assert runtime_positions[0] < wheel_positions[0] < runtime_positions[1] < wheel_positions[1] - - -def test_refresh_workflow_is_pr_safe_and_immutable(workflow): - before_pr_job, pr_job = workflow.split(" open-pull-request:", maxsplit=1) - trigger_paths = ( - ".github/workflows/refresh-build-dependencies.yml", - "eng/requirements-build-*.in", - "eng/requirements-build-*.txt", - "eng/requirements-test-linux.in", - "eng/requirements-test-linux.txt", - "requirements.txt", - "OneBranchPipelines/stages/build-*-single-stage.yml", - "OneBranchPipelines/stages/build-odbc-all-stage.yml", - ) - - assert "pull_request_target:" not in workflow - assert all(f" - {path}" in workflow for path in trigger_paths) - assert 'cron: "0 8 * * 1"' in workflow - assert "timezone: America/Los_Angeles" in workflow - assert "group: refresh-release-build-dependencies" in workflow - assert "cancel-in-progress: false" in workflow - assert "permissions:\n contents: read" in workflow - assert "contents: write" not in before_pr_job - assert "if: github.event_name != 'pull_request'" in pr_job - assert "contents: write" in pr_job - assert "pull-requests: write" in pr_job - assert workflow.count("persist-credentials: false") == 2 - assert workflow.count("github.event.pull_request.head.sha || 'main'") == 2 - - actions = ACTION.findall(workflow) - assert actions - assert all(re.fullmatch(r"[0-9a-f]{40}", revision) for _, revision in actions) - - -def test_refresh_workflow_compiles_and_verifies_committed_locks(workflow): - compile_jobs = _section(workflow, " compile-linux:", "\n validate-locks:") - compile_commands = [ - line.strip() for line in compile_jobs.splitlines() if "uv pip compile" in line - ] - conditional_upgrade = "${{ github.event_name != 'pull_request' && '--upgrade' || '' }}" - required_options = ( - conditional_upgrade, - "--generate-hashes", - "--no-emit-index-url", - "--no-header", - "--strip-extras", - "--python-version 3.10", - "--default-index https://pypi.org/simple", - ) - - assert len(compile_commands) == 3 - assert all( - all(option in command for option in required_options) for command in compile_commands - ) - assert "uv pip compile --upgrade" not in workflow - assert "eng/requirements-build-linux.txt eng/requirements-build-linux.in" in compile_commands[0] - assert "eng/requirements-test-linux.txt eng/requirements-test-linux.in" in compile_commands[1] - assert '"${{ matrix.output }}" "${{ matrix.input }}"' in compile_commands[2] - - for name, runner, input_path, output_path in ( - ( - "macos", - "macos-latest", - "eng/requirements-build-macos.in", - "eng/requirements-build-macos.txt", - ), - ( - "windows", - "windows-latest", - "eng/requirements-build-windows.in", - "eng/requirements-build-windows.txt", - ), - ( - "odbc", - "windows-latest", - "eng/requirements-build-odbc.in", - "eng/requirements-build-odbc.txt", - ), - ): - item = _matrix_item(compile_jobs, name) - assert f"os: {runner}" in item - assert f"input: {input_path}" in item - assert f"output: {output_path}" in item - - assert compile_jobs.count("if: github.event_name == 'pull_request'") == 2 - assert "git ls-files --error-unmatch -- eng/requirements-build-linux.txt" in compile_jobs - assert 'git ls-files --error-unmatch -- "${{ matrix.output }}"' in compile_jobs - assert "git diff --exit-code -- eng/requirements-build-linux.txt" in compile_jobs - assert 'git diff --exit-code -- "${{ matrix.output }}"' in compile_jobs - - -@pytest.mark.parametrize("name, expected", VALIDATION_MATRIX.items()) -def test_refresh_workflow_validates_each_release_target(workflow, name, expected): - validation = _section(workflow, " validate-locks:", "\n open-pull-request:") - artifact, lock, versions, platforms = expected - item = _matrix_item(validation, name) - - assert f"artifact: {artifact}" in item - assert f"lock: {lock}" in item - assert f'versions: "{versions}"' in item - assert f'platforms: "{platforms}"' in item - - for option in ( - "--dry-run", - "--no-cache", - "--only-binary :all:", - "--require-hashes", - '--python-version "$version"', - '--python-platform "$platform"', - ): - assert option in validation - - -def test_refresh_workflow_opens_one_tracked_update_pr(workflow): - pr_job = _section(workflow, " open-pull-request:") - - assert "needs: validate-locks" in pr_job - assert "ref: main" in pr_job - assert "BUILD_DEPENDENCY_WORK_ITEM:" in pr_job - assert "> AB#${BUILD_DEPENDENCY_WORK_ITEM}" in pr_job - assert 'branch="automation/refresh-build-dependencies"' in pr_job - assert "git diff --quiet -- eng/requirements-build-*.txt" in pr_job - assert '--force-with-lease="refs/heads/$branch:$remote_sha"' in pr_job - pr_lookup = ( - 'pr_count="$(gh pr list --head "$branch" --base main ' - '--state open --json number --jq length)"' - ) - assert pr_lookup in pr_job - assert 'if [ "$pr_count" = "0" ]; then' in pr_job - assert 'if [ "$(gh pr list' not in pr_job - assert "gh pr create \\" in pr_job From c09c7604649a6a569be124c9bb3aefd1c84fd8c8 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:40:08 +0530 Subject: [PATCH 03/12] DOC: Describe independent Conda release proposals Keep native packaging and release tooling independently mergeable to main without a stack or required merge order. Preserve candidate availability and production prerequisite caveats. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 14 ++++++++------ conda/README.md | 5 +++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ce44847d..b5890117 100644 --- a/README.md +++ b/README.md @@ -83,11 +83,13 @@ conda install -c "" -c microsoft -c conda-forge --strict-chan conda install -c "" -c microsoft -c defaults --override-channels "mssql-python=" ``` -**Conda release maintainers:** The dependent [release-additions PR](https://github.com/microsoft/mssql-python/pull/720) -supplies `OneBranchPipelines/conda-release-pipeline.yml` and the publication/provenance -validators; they are not part of this native-packaging change. The workflow described -below requires those additions and does not establish that production setup or publication -has occurred. Its default is `publishToConda=false`. +**Conda release maintainers:** Planned release tooling is proposed separately in the +[release-additions PR](https://github.com/microsoft/mssql-python/pull/720), including +`OneBranchPipelines/conda-release-pipeline.yml` and the publication/provenance validators. +These native-packaging changes do not provide that workflow or require a particular merge +order. The workflow described below applies only when that separate tooling is available; +it does not establish that production setup or publication has occurred. +Its default is `publishToConda=false`. Select the exact completed Conda build and expected package version. The pipeline verifies its recorded upstream wheel run, checks the 28-package matrix and ELF/PE/Mach-O payloads, and logs archive SHA-256 values and the upload plan without publishing. @@ -104,7 +106,7 @@ read access to group/check configuration and run-check evidence. Record the inst IDs as non-secret group variables `CONDA_PUBLICATION_LOCK_CHECK_ID` and `CONDA_PUBLICATION_APPROVAL_CHECK_ID`; absent or mismatched IDs block publication. Every writer to `microsoft/mssql-python` label `main` must use this same protected resource; other credentials -or pipelines must not bypass it. The dependent workflow's publish-only `CondaRelease` stage references this group +or pipelines must not bypass it. The proposed workflow's publish-only `CondaRelease` stage references this group with `lockBehavior: sequential`, and refuses upload or promotion without matching, successful current-stage checks. YAML `lockBehavior` alone does not create the lock. The server-enforced stage lock must cover snapshot, upload, promotion, rollback, and cleanup. diff --git a/conda/README.md b/conda/README.md index 5b2251eb..5c110087 100644 --- a/conda/README.md +++ b/conda/README.md @@ -33,7 +33,8 @@ configuration remain external prerequisites. Use only organizationally approved channels and handle applicable terms separately; the Windows ARM64 dependency profile includes Anaconda `defaults`. Run this existing build workflow only in a disposable isolated installation: shared-environment -ownership hardening is outside this change. The dependent release-additions PR supplies -the proposed publication/provenance gates; see the [release-maintainer prerequisites](../README.md#installation). +ownership hardening is outside this change. Publication/provenance tooling is proposed +in a separate release-additions PR, without a required merge order; see the +[release-maintainer prerequisites](../README.md#installation). Neither this native-packaging change nor validate-only success authorizes production publication. From c5254522f0ca76b0305f1520c2228a5079a293e6 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:46:57 +0530 Subject: [PATCH 04/12] CHORE: Drop unused PyYAML test dependency additions The remaining tests and release tooling no longer consume PyYAML. Restore the three requirements files exactly to main, retaining all native packaging fixes and pipeline-only test removals. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/requirements-build-macos.txt | 75 -------------------------------- eng/requirements-test-linux.txt | 75 -------------------------------- requirements.txt | 1 - 3 files changed, 151 deletions(-) diff --git a/eng/requirements-build-macos.txt b/eng/requirements-build-macos.txt index 6a1f26f2..e1e3f6f2 100644 --- a/eng/requirements-build-macos.txt +++ b/eng/requirements-build-macos.txt @@ -1240,81 +1240,6 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black -pyyaml==6.0.3 \ - --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ - --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ - --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ - --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ - --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ - --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ - --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ - --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ - --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ - --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ - --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ - --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ - --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ - --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ - --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ - --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ - --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ - --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ - --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ - --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ - --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ - --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ - --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ - --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ - --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ - --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ - --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ - --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ - --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ - --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ - --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ - --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ - --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ - --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ - --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ - --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ - --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ - --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ - --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ - --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ - --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ - --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ - --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ - --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ - --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ - --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ - --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ - --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ - --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ - --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ - --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ - --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ - --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ - --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ - --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ - --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ - --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ - --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ - --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ - --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ - --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ - --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ - --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ - --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ - --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ - --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ - --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ - --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ - --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ - --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ - --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ - --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ - --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 - # via -r eng/../requirements.txt requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed diff --git a/eng/requirements-test-linux.txt b/eng/requirements-test-linux.txt index 37610921..ef1d0a82 100644 --- a/eng/requirements-test-linux.txt +++ b/eng/requirements-test-linux.txt @@ -1230,81 +1230,6 @@ pytokens==0.4.1 \ --hash=sha256:ee44d0f85b803321710f9239f335aafe16553b39106384cef8e6de40cb4ef2f6 \ --hash=sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324 # via black -pyyaml==6.0.3 \ - --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ - --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ - --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ - --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ - --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ - --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ - --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ - --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ - --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ - --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ - --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ - --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ - --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ - --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ - --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ - --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ - --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ - --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ - --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ - --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ - --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ - --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ - --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ - --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ - --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ - --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ - --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ - --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ - --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ - --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ - --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ - --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ - --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ - --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ - --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ - --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ - --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ - --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ - --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ - --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ - --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ - --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ - --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ - --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ - --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ - --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ - --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ - --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ - --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ - --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ - --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ - --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ - --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ - --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ - --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ - --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ - --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ - --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ - --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ - --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ - --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ - --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ - --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ - --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ - --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ - --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ - --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ - --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ - --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ - --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ - --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ - --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ - --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 - # via -r eng/../requirements.txt requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed diff --git a/requirements.txt b/requirements.txt index 2164be12..b028797a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ # Testing dependencies pytest pytest-cov -PyYAML zstandard coverage unittest-xml-reporting From 9ebf192d2d70e8ddaac91dcb0eb8a8124b707062 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:56:10 +0530 Subject: [PATCH 05/12] FIX: Preserve native core package completeness Include exact stable-ABI core filenames in wheels and require the core package initializer in cross-platform archive audits. Add focused behavioral regressions and trim unrelated release administration detail. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 39 +++--------------- eng/scripts/_conda_pkg.py | 7 +++- setup.py | 2 + tests/test_029_bundled_binary_audit.py | 16 +++++--- tests/test_030_pe_machine_assert.py | 55 +++++++++++++++++++++++++- tests/test_035_conda_macho_assert.py | 11 +++++- 6 files changed, 88 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index b5890117..5638fd89 100644 --- a/README.md +++ b/README.md @@ -83,39 +83,12 @@ conda install -c "" -c microsoft -c conda-forge --strict-chan conda install -c "" -c microsoft -c defaults --override-channels "mssql-python=" ``` -**Conda release maintainers:** Planned release tooling is proposed separately in the -[release-additions PR](https://github.com/microsoft/mssql-python/pull/720), including -`OneBranchPipelines/conda-release-pipeline.yml` and the publication/provenance validators. -These native-packaging changes do not provide that workflow or require a particular merge -order. The workflow described below applies only when that separate tooling is available; -it does not establish that production setup or publication has occurred. -Its default is `publishToConda=false`. -Select the exact completed Conda build and expected package version. -The pipeline verifies its recorded upstream wheel run, checks the 28-package matrix and -ELF/PE/Mach-O payloads, and logs archive SHA-256 values and the upload plan without publishing. -Feature-branch candidates are allowed only for validation; production requires release, -Conda producer, and wheel sources on `refs/heads/main`. Recipe and wheel commits may differ, -but each must match its authoritative run record. This is static release readiness, not -live SQL/TLS, bulk-copy, or Arrow feature certification. - -**Production prerequisite (administrator setup, not performed by a dry run):** in the -`SqlClientDrivers/mssql-python` ADO project, protect the existing **Anaconda Publishing** -variable group **117** with an enabled native **Exclusive lock** check and a designated -**Approval** check. Authorize only release definition **2322**, and grant its build identity -read access to group/check configuration and run-check evidence. Record the installed check -IDs as non-secret group variables `CONDA_PUBLICATION_LOCK_CHECK_ID` and -`CONDA_PUBLICATION_APPROVAL_CHECK_ID`; absent or mismatched IDs block publication. Every writer to -`microsoft/mssql-python` label `main` must use this same protected resource; other credentials -or pipelines must not bypass it. The proposed workflow's publish-only `CondaRelease` stage references this group -with `lockBehavior: sequential`, and refuses upload or promotion without matching, -successful current-stage checks. YAML `lockBehavior` alone does not create the lock. -The server-enforced stage lock must cover snapshot, upload, promotion, rollback, and cleanup. -Rollback is compensating, not atomic; a killed process can leave partial labels, which a -subsequent authorized run must re-verify. Metadata/label API calls use 15-second connect -and 60-second read timeouts; the publishing job, including CLI uploads, is capped at 60 minutes. -Until resource setup and a controlled positive -lock/approval evaluation are verified, production remains blocked; validate-only success -does not prove or authorize production publication. +**Conda release status:** Publication tooling and its administrator prerequisites are +proposed separately in the [release-additions PR](https://github.com/microsoft/mssql-python/pull/720). +These native-packaging changes do not publish packages or require a particular merge order. +Static audits and import checks do not certify SQL, certificate-verified TLS, authentication, +bulk copy, or optional features across the full matrix. Production publication remains gated +on separate release controls and qualification; validate-only success does not authorize it. ## Key Features ### Supported Platforms diff --git a/eng/scripts/_conda_pkg.py b/eng/scripts/_conda_pkg.py index 3ed2d23d..39c3f0f1 100644 --- a/eng/scripts/_conda_pkg.py +++ b/eng/scripts/_conda_pkg.py @@ -23,7 +23,7 @@ def validate_native_contract( members: Iterable[tuple[str, bytes]], index: dict[str, Any] ) -> list[str]: - """Require a target binding and importable core filename; platform audits check headers. + """Require a target binding, core extension and initializer; platform audits check headers. Wheels may include bindings for several Python minors. The core uses Python's normal extension loader, including its stable-ABI suffix. These static checks @@ -71,6 +71,11 @@ def validate_native_contract( else (f"mssql_py_core.cpython-{abi[2]}-{arch}.so", "mssql_py_core.abi3.so") ) errors = [] + initializer = f"{root}mssql_py_core/__init__.py" + if names.count(initializer) != 1: + errors.append( + f"expected exactly one required {initializer}; found {names.count(initializer)}" + ) if len(bindings) != 1: errors.append( f"expected exactly one normal cp{abi[2]} native binding; found {len(bindings)}" diff --git a/setup.py b/setup.py index c3e17cb9..76b580da 100644 --- a/setup.py +++ b/setup.py @@ -199,6 +199,8 @@ def run(self): "mssql_py_core": [ "mssql_py_core.cp*.pyd", "mssql_py_core.cp*.so", + "mssql_py_core.pyd", + "mssql_py_core.abi3.so", ], } diff --git a/tests/test_029_bundled_binary_audit.py b/tests/test_029_bundled_binary_audit.py index 26f18f39..835b0190 100644 --- a/tests/test_029_bundled_binary_audit.py +++ b/tests/test_029_bundled_binary_audit.py @@ -157,6 +157,7 @@ def add_str(s): ] _BINDING = "lib/python3.12/site-packages/mssql_python/ddbc_bindings.cp312-x86_64.so" _CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-x86_64-linux-gnu.so" +_CORE_INIT = "lib/python3.12/site-packages/mssql_py_core/__init__.py" _DISTROS_BY_SUBDIR = { "linux-64": ("alpine", "debian_ubuntu", "rhel", "suse"), "linux-aarch64": ("alpine", "debian_ubuntu", "rhel"), @@ -201,6 +202,7 @@ def add(name, data): extension_arch = "aarch64" if subdir == "linux-aarch64" else "x86_64" if native_payload is None: native_payload = { + _CORE_INIT: b"from .mssql_py_core import *\n", _BINDING.replace("x86_64", extension_arch): _make_elf64(machine=machine), _CORE.replace("x86_64", extension_arch): _make_elf64( machine=machine, versions=("GLIBC_2.2.5", "GLIBC_2.34") @@ -268,9 +270,9 @@ def test_audit_passes_with_exact_climb(tmp_path): assert audit.audit_package(_make_pkg(tmp_path)) == [] -@pytest.mark.parametrize("missing", [_BINDING, _CORE]) +@pytest.mark.parametrize("missing", [_BINDING, _CORE, _CORE_INIT]) def test_full_feature_package_requires_binding_and_core(tmp_path, missing): - native = {_BINDING: _make_elf64(), _CORE: _make_elf64()} + native = {_BINDING: _make_elf64(), _CORE: _make_elf64(), _CORE_INIT: b""} del native[missing] errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) assert any("exactly one" in error for error in errors) @@ -278,7 +280,7 @@ def test_full_feature_package_requires_binding_and_core(tmp_path, missing): @pytest.mark.parametrize("component", [_BINDING, _CORE]) def test_every_required_extension_checks_elf_arch_not_filename(tmp_path, component): - native = {_BINDING: _make_elf64(), _CORE: _make_elf64()} + native = {_BINDING: _make_elf64(), _CORE: _make_elf64(), _CORE_INIT: b""} native[component] = _make_elf64(machine=183) errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) assert any(component in error and "does not match" in error for error in errors) @@ -287,7 +289,7 @@ def test_every_required_extension_checks_elf_arch_not_filename(tmp_path, compone @pytest.mark.parametrize("component", [_BINDING, _CORE]) @pytest.mark.parametrize("wrong_tag", ["311", "312t", "312d"]) def test_required_extensions_reject_wrong_or_non_normal_abi(tmp_path, component, wrong_tag): - native = {_BINDING: _make_elf64(), _CORE: _make_elf64()} + native = {_BINDING: _make_elf64(), _CORE: _make_elf64(), _CORE_INIT: b""} data = native.pop(component) native[component.replace("312", wrong_tag)] = data assert audit.audit_package(_make_pkg(tmp_path, native_payload=native)) @@ -295,6 +297,7 @@ def test_required_extensions_reject_wrong_or_non_normal_abi(tmp_path, component, def test_core_abi3_and_extra_normal_bindings_are_supported(tmp_path): native = { + _CORE_INIT: b"from .mssql_py_core import *\n", _BINDING: _make_elf64(), _BINDING.replace("312", "310"): _make_elf64(), _CORE.replace("cpython-312-x86_64-linux-gnu", "abi3"): _make_elf64(), @@ -332,6 +335,7 @@ def test_core_symbol_floor_compatible_with_archive_minimum(tmp_path, declared): def test_symbol_floor_is_read_from_version_needs_not_arbitrary_bytes(tmp_path): native = { + _CORE_INIT: b"from .mssql_py_core import *\n", _BINDING: _make_elf64(), _CORE: _make_elf64(versions=("GLIBC_2.34",)) + b"GLIBC_99.99\x00", } @@ -341,6 +345,7 @@ def test_symbol_floor_is_read_from_version_needs_not_arbitrary_bytes(tmp_path): def test_auxiliary_native_library_symbol_floor_is_checked(tmp_path): extra = "lib/python3.12/site-packages/mssql_py_core.libs/libsupport.so.1" native = { + _CORE_INIT: b"from .mssql_py_core import *\n", _BINDING: _make_elf64(), _CORE: _make_elf64(), extra: _make_elf64(versions=("GLIBC_2.35",)), @@ -372,7 +377,7 @@ def test_malformed_required_core_elf_cannot_pass(tmp_path, damage): struct.pack_into(" bytes: @@ -129,6 +132,42 @@ def test_zstd_backend_is_available_for_conda_audit_tests(): ) +def test_wheel_retains_normal_and_stable_abi_core_extensions(tmp_path): + shutil.copy2(_MODULE_PATH.parents[2] / "setup.py", tmp_path / "setup.py") + sources = { + "PyPI_Description.md": "Packaging fixture", + "mssql_python/__init__.py": "", + "mssql_python_odbc/__init__.py": '__version__ = "18.6.2.1"\n', + "mssql_py_core/__init__.py": "from .mssql_py_core import *\n", + } + for relative, content in sources.items(): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + extensions = ( + "mssql_py_core.cp312-win_arm64.pyd", + "mssql_py_core.cpython-312-x86_64-linux-gnu.so", + "mssql_py_core.pyd", + "mssql_py_core.abi3.so", + ) + for name in extensions: + (tmp_path / "mssql_py_core" / name).write_bytes(b"native payload fixture") + + result = subprocess.run( + [sys.executable, "setup.py", "--quiet", "bdist_wheel"], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stdout + result.stderr + wheels = list((tmp_path / "dist").glob("*.whl")) + assert len(wheels) == 1 + with zipfile.ZipFile(wheels[0]) as wheel: + for name in extensions: + assert wheel.read(f"mssql_py_core/{name}") == b"native payload fixture" + + def _make_conda(tmp_path, subdir, payload, depends=None): """Build a minimal .conda (info-*.tar.zst + pkg-*.tar.zst) with the given payload files.""" name = "mssql-python-1.13.0-py312_0" @@ -167,6 +206,7 @@ def test_win_arm64_arm64_binaries_pass(tmp_path): tmp_path, "win-arm64", { + _CORE_INIT: b"from .mssql_py_core import *\n", "Lib/site-packages/mssql_python/ddbc_bindings.cp312-arm64.pyd": _fake_pe(_ARM64), "Lib/site-packages/mssql_py_core/mssql_py_core.cp312-win_arm64.pyd": _fake_pe(_ARM64), "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/msodbcsql18.dll": _fake_pe( @@ -181,7 +221,17 @@ def test_win_arm64_arm64_binaries_pass(tmp_path): @pytest.mark.parametrize( - "state", ["valid", "missing", "wrong-arch", "wrong-tag", "abi3", "missing-abi", "wrong-abi"] + "state", + [ + "valid", + "missing", + "missing-init", + "wrong-arch", + "wrong-tag", + "abi3", + "missing-abi", + "wrong-abi", + ], ) def test_required_core_contract(tmp_path, state): pin = "python_abi 3.12.* *_cp312" @@ -191,6 +241,7 @@ def test_required_core_contract(tmp_path, state): depends.append(pin if state != "wrong-abi" else "python_abi 3.12.* *_cp313") core = "Lib/site-packages/mssql_py_core/mssql_py_core.cp312-win_arm64.pyd" payload = { + _CORE_INIT: b"from .mssql_py_core import *\n", "Lib/site-packages/mssql_python/ddbc_bindings.cp312-arm64.pyd": _fake_pe(_ARM64), core: _fake_pe(_ARM64), "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/msodbcsql18.dll": _fake_pe(_ARM64), @@ -198,6 +249,8 @@ def test_required_core_contract(tmp_path, state): } if state == "missing": del payload[core] + elif state == "missing-init": + del payload[_CORE_INIT] elif state == "wrong-arch": payload[core] = _fake_pe(_AMD64) elif state == "wrong-tag": diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index 8532a0cb..31e2bba7 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -162,6 +162,7 @@ def _make_conda(tmp_path, subdir, payload): _BINDING = "lib/python3.12/site-packages/mssql_python/ddbc_bindings.cp312-darwin.so" _CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-darwin.so" +_CORE_INIT = "lib/python3.12/site-packages/mssql_py_core/__init__.py" _DRIVER_ROOT = "lib/python3.12/site-packages/mssql_python_odbc/libs/macos" _DRIVER_LIBRARIES = ( "libltdl.7.dylib", @@ -175,7 +176,11 @@ def _realistic_payload(binding, arm64=None, x86_64=None): """Mirror the wheel's two architecture-specific four-library driver directories.""" arm64 = arm64 or _fake_macho_thin(_ARM64) x86_64 = x86_64 or _fake_macho_thin(_X86_64) - payload = {_BINDING: binding, _CORE: _fake_macho_fat([_X86_64, _ARM64])} + payload = { + _BINDING: binding, + _CORE: _fake_macho_fat([_X86_64, _ARM64]), + _CORE_INIT: b"from .mssql_py_core import *\n", + } for library in _DRIVER_LIBRARIES: payload[f"{_DRIVER_ROOT}/arm64/lib/{library}"] = arm64 payload[f"{_DRIVER_ROOT}/x86_64/lib/{library}"] = x86_64 @@ -194,11 +199,13 @@ def test_osx_packages_accept_real_split_driver_layout(tmp_path, subdir): assert mac.audit_package(p) == [] -@pytest.mark.parametrize("state", ["missing", "wrong-arch", "wrong-tag", "abi3"]) +@pytest.mark.parametrize("state", ["missing", "missing-init", "wrong-arch", "wrong-tag", "abi3"]) def test_required_core_contract(tmp_path, state): payload = _realistic_payload(_fake_macho_fat([_X86_64, _ARM64])) if state == "missing": del payload[_CORE] + elif state == "missing-init": + del payload[_CORE_INIT] elif state == "wrong-arch": payload[_CORE] = _fake_macho_thin(_X86_64) elif state == "wrong-tag": From 5a99c7e7ae5effe68b0cfd89d6293aabf47860fe Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 10:59:59 +0530 Subject: [PATCH 06/12] FIX: Require an explicit Conda wheel version Remove the recipe's unrelated release-version fallback. Preserve the orchestrator's selected-wheel version input and verify strict recipe rendering fails clearly when it is missing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- conda/README.md | 6 +++++- conda/mssql-python/meta.yaml | 2 +- tests/test_034_conda_verify_cwd.py | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/conda/README.md b/conda/README.md index 5c110087..1652e119 100644 --- a/conda/README.md +++ b/conda/README.md @@ -5,6 +5,10 @@ Conda package**, including the required bulk-copy core. Pip instead installs `mssql-python-odbc` as a separate companion distribution. Neither requires a separately installed ODBC driver or driver manager. +Direct recipe builds must set `MSSQL_PYTHON_VERSION` to the exact selected code-wheel +version before rendering/building. The shared orchestrator derives and supplies it +automatically; omitted input fails recipe rendering instead of choosing a release. + This is a temporary candidate, not an announcement of public channel availability. Obtain the exact candidate archive/channel from its owner and install into a new Conda environment. Activate it and select the same interpreter/kernel in your IDE @@ -35,6 +39,6 @@ the Windows ARM64 dependency profile includes Anaconda `defaults`. Run this exis build workflow only in a disposable isolated installation: shared-environment ownership hardening is outside this change. Publication/provenance tooling is proposed in a separate release-additions PR, without a required merge order; see the -[release-maintainer prerequisites](../README.md#installation). +[release status and qualification caveats](../README.md#installation). Neither this native-packaging change nor validate-only success authorizes production publication. diff --git a/conda/mssql-python/meta.yaml b/conda/mssql-python/meta.yaml index 4ede11dd..fb02fb7f 100644 --- a/conda/mssql-python/meta.yaml +++ b/conda/mssql-python/meta.yaml @@ -1,4 +1,4 @@ -{% set version = environ.get('MSSQL_PYTHON_VERSION', '1.14.0') %} +{% set version = environ['MSSQL_PYTHON_VERSION'] %} package: name: mssql-python diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index 9e7570c1..b3d406e0 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -441,6 +441,20 @@ def test_build_env_sets_subdir_for_cross_build(monkeypatch): assert env["CONDA_SUBDIR"] == "osx-arm64" +def test_recipe_requires_explicit_wheel_version(): + jinja2 = pytest.importorskip("jinja2", reason="Conda recipe rendering requires Jinja2") + recipe = _ORCH_PATH.parents[2] / "conda" / "mssql-python" / "meta.yaml" + template = jinja2.Environment(undefined=jinja2.StrictUndefined).from_string( + recipe.read_text(encoding="utf-8") + ) + env = _load_orchestrator().build_env("1.2.3", "18.6.2", "wheels", "") + assert 'version: "1.2.3"' in template.render(environ=env) + + del env["MSSQL_PYTHON_VERSION"] + with pytest.raises(jinja2.UndefinedError, match="MSSQL_PYTHON_VERSION"): + template.render(environ=env) + + def test_win_arm64_real_environment_create_failure_is_blocking(tmp_path, monkeypatch): """A successful solve does not prove package extraction/linking succeeds.""" mod = _load_orchestrator() From b0f382a7ce3af822d6bdcab9c94cc38b89575b93 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 11:03:50 +0530 Subject: [PATCH 07/12] FIX: Exclude inherited automatic channel terms acceptance Remove the opt-in variable from the copied build environment while preserving the caller's environment. Cover both unset and inherited opt-in states with the existing behavioral regression. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- OneBranchPipelines/scripts/build_conda_packages.py | 1 + tests/test_034_conda_verify_cwd.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/scripts/build_conda_packages.py b/OneBranchPipelines/scripts/build_conda_packages.py index d8d79c3c..945761a7 100644 --- a/OneBranchPipelines/scripts/build_conda_packages.py +++ b/OneBranchPipelines/scripts/build_conda_packages.py @@ -297,6 +297,7 @@ def build_env( ) -> dict[str, str]: """The environment consumed by the recipe (jinja + build.sh/bld.bat) and by conda-build.""" env = dict(os.environ) + env.pop("CONDA_PLUGINS_AUTO_ACCEPT_TOS", None) env["WHEELS_DIR"] = links env["MSSQL_PYTHON_VERSION"] = mssql_ver env["MSSQL_ODBC_VERSION"] = odbc_ver diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index b3d406e0..a00d5b0e 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -661,12 +661,17 @@ def test_both_windows_targets_run_native_audit(subdir, monkeypatch): assert pe_calls[0][-2:] == ["--subdir", subdir] -def test_build_does_not_automatically_accept_channel_terms(monkeypatch): +@pytest.mark.parametrize("inherited", [None, "true"]) +def test_build_does_not_automatically_accept_channel_terms(monkeypatch, inherited): mod = _load_orchestrator() - monkeypatch.delenv("CONDA_PLUGINS_AUTO_ACCEPT_TOS", raising=False) + if inherited is None: + monkeypatch.delenv("CONDA_PLUGINS_AUTO_ACCEPT_TOS", raising=False) + else: + monkeypatch.setenv("CONDA_PLUGINS_AUTO_ACCEPT_TOS", inherited) assert "CONDA_PLUGINS_AUTO_ACCEPT_TOS" not in mod.build_env( "1.14.0", "18.6.2.1", "wheels", "win-arm64" ) + assert os.environ.get("CONDA_PLUGINS_AUTO_ACCEPT_TOS") == inherited @pytest.mark.parametrize("subdir", ["win-64", "win-arm64"]) From 642f6e2ec20b5f902d72adee9706788740a4591d Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 11:06:42 +0530 Subject: [PATCH 08/12] FIX: Respect optional Mach-O test compression backend Apply the existing archive-backend skip guard to the core contract cases. All 25 Mach-O cases execute with zstandard; without a backend, seven parser cases pass and eighteen archive cases skip consistently. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_035_conda_macho_assert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index 31e2bba7..9e33a781 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -199,6 +199,7 @@ def test_osx_packages_accept_real_split_driver_layout(tmp_path, subdir): assert mac.audit_package(p) == [] +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") @pytest.mark.parametrize("state", ["missing", "missing-init", "wrong-arch", "wrong-tag", "abi3"]) def test_required_core_contract(tmp_path, state): payload = _realistic_payload(_fake_macho_fat([_X86_64, _ARM64])) From 8a6d6dd640426c49d84d07b24961ee57a305a0da Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 11:42:48 +0530 Subject: [PATCH 09/12] FIX: Skip wheel-build regression without its optional backend Driver-test environments do not all install the wheel build backend. Keep the real archive assertion active where wheel is available, including verified Windows and Ubuntu CI legs, without adding build dependencies to driver-only tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_030_pe_machine_assert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index 4db8d1c4..615bba2a 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -133,6 +133,7 @@ def test_zstd_backend_is_available_for_conda_audit_tests(): def test_wheel_retains_normal_and_stable_abi_core_extensions(tmp_path): + pytest.importorskip("wheel", reason="Wheel archive regression requires the wheel build backend") shutil.copy2(_MODULE_PATH.parents[2] / "setup.py", tmp_path / "setup.py") sources = { "PyPI_Description.md": "Packaging fixture", From 772f8180babbadb8d7fc14f13a13a94abce189ad Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 12:40:07 +0530 Subject: [PATCH 10/12] FIX: Match wheel test preflight to setuptools bootstrap Check the existing producer bootstrap before deciding that the wheel backend is unavailable. This retains real archive coverage with setuptools-vendored wheel without adding a dependency. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_030_pe_machine_assert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index 615bba2a..df2d28df 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -133,6 +133,7 @@ def test_zstd_backend_is_available_for_conda_audit_tests(): def test_wheel_retains_normal_and_stable_abi_core_extensions(tmp_path): + pytest.importorskip("setuptools", reason="Wheel archive regression requires setuptools") pytest.importorskip("wheel", reason="Wheel archive regression requires the wheel build backend") shutil.copy2(_MODULE_PATH.parents[2] / "setup.py", tmp_path / "setup.py") sources = { From 90df29dac6ed954dc69d3e5acdab0480adb32b88 Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 16:53:42 +0530 Subject: [PATCH 11/12] FIX: Reject malformed Conda metadata and incomplete direct recipe payloads Validate dependency field shape through the shared package reader. Require the core initializer and compatible extension after native and cross installation paths, while retaining the separate native header audits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- conda/README.md | 2 + conda/mssql-python/bld.bat | 17 +++-- conda/mssql-python/build.sh | 11 +++ eng/scripts/_conda_pkg.py | 15 ++-- tests/test_029_bundled_binary_audit.py | 13 +++- tests/test_030_pe_machine_assert.py | 86 +++++++++++++++++++++- tests/test_035_conda_macho_assert.py | 98 +++++++++++++++++++++++++- 7 files changed, 226 insertions(+), 16 deletions(-) diff --git a/conda/README.md b/conda/README.md index 1652e119..02329d34 100644 --- a/conda/README.md +++ b/conda/README.md @@ -8,6 +8,8 @@ separately installed ODBC driver or driver manager. Direct recipe builds must set `MSSQL_PYTHON_VERSION` to the exact selected code-wheel version before rendering/building. The shared orchestrator derives and supplies it automatically; omitted input fails recipe rendering instead of choosing a release. +Both native installation and cross extraction require the bulk-copy initializer and +a compatible extension filename. The separate native audits still validate binary headers. This is a temporary candidate, not an announcement of public channel availability. Obtain the exact candidate archive/channel from its owner and install into a new diff --git a/conda/mssql-python/bld.bat b/conda/mssql-python/bld.bat index 91dbc8e3..c0a41653 100644 --- a/conda/mssql-python/bld.bat +++ b/conda/mssql-python/bld.bat @@ -39,17 +39,22 @@ if errorlevel 1 ( echo ERROR: extracted "!CODE_WHL!" has no mssql_python\ddbc_bindings.cp%CONDA_PY% pyd ^(wrong-Python binding^). exit /b 1 ) - REM Never silently drop bulk copy. Bare .pyd is the Windows stable-ABI suffix; - REM the package's PE audit checks actual architecture before staging. - if not exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" if not exist "%SP%\mssql_py_core\mssql_py_core.pyd" ( - echo ERROR: required mssql_py_core is missing or incompatible with cp%CONDA_PY% !ODBC_ARCH!. Use a corrected upstream wheel; refusing reduced functionality. - exit /b 1 - ) ) else ( "%PYTHON%" -m pip install --no-deps --no-index --find-links "%WHEELS_DIR%" %PKG_NAME%==%PKG_VERSION% -vv if errorlevel 1 exit /b 1 ) +REM Both install paths require bulk copy. The PE audit still checks actual architecture. +if not exist "%SP%\mssql_py_core\__init__.py" ( + echo ERROR: required mssql_py_core initializer is missing. Use a corrected upstream wheel; refusing reduced functionality. + exit /b 1 +) +REM Bare .pyd is the Windows stable-ABI suffix. +if not exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" if not exist "%SP%\mssql_py_core\mssql_py_core.pyd" ( + echo ERROR: required mssql_py_core is missing or incompatible with cp%CONDA_PY% !ODBC_ARCH!. Use a corrected upstream wheel; refusing reduced functionality. + exit /b 1 +) + REM Extract the arch-specific odbc wheel into the SAME site-packages so REM mssql_python_odbc\libs\ sits beside mssql_python\ and the loader finds the driver. REM The py3-none tag only means "no Python bytecode" -- the vendored driver DLLs ARE diff --git a/conda/mssql-python/build.sh b/conda/mssql-python/build.sh index 2bf1733d..66b5adb8 100644 --- a/conda/mssql-python/build.sh +++ b/conda/mssql-python/build.sh @@ -18,7 +18,9 @@ odbc_ver="${MSSQL_ODBC_VERSION:?MSSQL_ODBC_VERSION not set}" if "$PYTHON" -c "import sys" >/dev/null 2>&1; then "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "$PKG_NAME==$PKG_VERSION" -vv "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "mssql-python-odbc==$odbc_ver" -vv + core_suffix="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))')" else + core_suffix=".cpython-${CONDA_PY}-darwin.so" echo "Host Python '$PYTHON' is not executable on this agent (non-emulated cross-build);" echo "extracting both wheels into \$SP_DIR without running Python." mkdir -p "$SP_DIR" @@ -54,6 +56,15 @@ else unzip -oq "$odbc_whl" -d "$SP_DIR" fi +# Both install paths require bulk copy; the platform audits still check binary headers. +if [ ! -f "$SP_DIR/mssql_py_core/__init__.py" ] || { + [ ! -f "$SP_DIR/mssql_py_core/mssql_py_core${core_suffix}" ] && + [ ! -f "$SP_DIR/mssql_py_core/mssql_py_core.abi3.so" ] +}; then + echo "ERROR: required mssql_py_core initializer or compatible extension is missing. Use a corrected upstream wheel; refusing reduced functionality." >&2 + exit 1 +fi + # --------------------------------------------------------------------------- # Linux driver reachability (#563) -- the core fix. # --------------------------------------------------------------------------- diff --git a/eng/scripts/_conda_pkg.py b/eng/scripts/_conda_pkg.py index 39c3f0f1..8bed56d1 100644 --- a/eng/scripts/_conda_pkg.py +++ b/eng/scripts/_conda_pkg.py @@ -157,11 +157,18 @@ def read_index(path: str) -> dict[str, Any]: member = tf.extractfile("info/index.json") if member is None: raise ValueError("info/index.json missing") - return json.load(member) - if path.endswith(".tar.bz2"): + index = json.load(member) + elif path.endswith(".tar.bz2"): with tarfile.open(path, "r:bz2") as tf: member = tf.extractfile("info/index.json") if member is None: raise ValueError("info/index.json missing") - return json.load(member) - raise ValueError("unrecognized conda package extension") + index = json.load(member) + else: + raise ValueError("unrecognized conda package extension") + if not isinstance(index, dict): + raise ValueError("info/index.json must be an object") + depends = index.get("depends", []) + if not isinstance(depends, list) or any(not isinstance(dep, str) for dep in depends): + raise ValueError("info/index.json depends must be a list of dependency strings") + return index diff --git a/tests/test_029_bundled_binary_audit.py b/tests/test_029_bundled_binary_audit.py index 835b0190..c096e9ba 100644 --- a/tests/test_029_bundled_binary_audit.py +++ b/tests/test_029_bundled_binary_audit.py @@ -170,7 +170,7 @@ def _make_pkg( rpath=None, subdir="linux-64", vendored=None, - depends=None, + depends=tuple(_GOOD_DEPENDS), driver_needed=None, inst_needed=None, machine=62, @@ -194,7 +194,7 @@ def add(name, data): "version": "1.13.0", "build": "py312_0", "subdir": subdir, - "depends": _GOOD_DEPENDS if depends is None else depends, + "depends": depends, } ).encode(), ) @@ -241,6 +241,15 @@ def add(name, data): # --- low-level parser ------------------------------------------------------- +@pytest.mark.parametrize( + "depends", + [None, 17, "python_abi 3.12.* *_cp312", {"python_abi": "3.12"}, ["python_abi", None]], +) +def test_audit_reports_malformed_dependency_field(tmp_path, depends): + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("malformed" in error and "depends" in error for error in errors) + + def test_elf_dynamic_pt_parse(): data = _make_elf64(runpath=_GOOD_RUNPATH, needed=["libkrb5.so.3", "libodbcinst.so.2"]) dyn = audit.elf_dynamic(data) diff --git a/tests/test_030_pe_machine_assert.py b/tests/test_030_pe_machine_assert.py index df2d28df..f5b43b86 100644 --- a/tests/test_030_pe_machine_assert.py +++ b/tests/test_030_pe_machine_assert.py @@ -9,10 +9,12 @@ import importlib.util import io import json +import os import shutil import struct import subprocess import sys +import sysconfig import tarfile import zipfile from pathlib import Path @@ -170,7 +172,81 @@ def test_wheel_retains_normal_and_stable_abi_core_extensions(tmp_path): assert wheel.read(f"mssql_py_core/{name}") == b"native payload fixture" -def _make_conda(tmp_path, subdir, payload, depends=None): +@pytest.mark.skipif(sys.platform != "win32", reason="Windows recipe requires cmd.exe") +@pytest.mark.parametrize("cross_build", [False, True]) +@pytest.mark.parametrize("state", ["valid", "missing", "missing-init", "wrong-tag", "abi3"]) +def test_windows_recipe_requires_core_on_both_install_paths(tmp_path, cross_build, state): + wheels = tmp_path / "wheels" + wheels.mkdir() + prefix = tmp_path / "prefix" + site_packages = prefix / "Lib" / "site-packages" + tag = f"{sys.version_info.major}{sys.version_info.minor}" + arch = sysconfig.get_platform().replace("-", "_") + machine = _ARM64 if arch == "win_arm64" else _AMD64 + core = f"mssql_py_core/mssql_py_core.cp{tag}-{arch}.pyd" + payload = { + "mssql_python/__init__.py": b"", + f"mssql_python/ddbc_bindings.cp{tag}-{arch}.pyd": _fake_pe(machine), + "mssql_py_core/__init__.py": b"from .mssql_py_core import *\n", + core: _fake_pe(machine), + } + if state == "missing": + del payload[core] + elif state == "missing-init": + del payload["mssql_py_core/__init__.py"] + elif state == "wrong-tag": + payload[core.replace(f".cp{tag}-", ".cp999-")] = payload.pop(core) + elif state == "abi3": + payload["mssql_py_core/mssql_py_core.pyd"] = payload.pop(core) + dist_info = "mssql_python-1.13.0.dist-info" + payload[f"{dist_info}/METADATA"] = ( + b"Metadata-Version: 2.1\nName: mssql-python\nVersion: 1.13.0\n" + ) + payload[f"{dist_info}/WHEEL"] = ( + f"Wheel-Version: 1.0\nRoot-Is-Purelib: false\nTag: cp{tag}-cp{tag}-{arch}\n".encode() + ) + payload[f"{dist_info}/RECORD"] = "\n".join(f"{name},," for name in payload).encode() + with zipfile.ZipFile(wheels / f"mssql_python-1.13.0-cp{tag}-cp{tag}-{arch}.whl", "w") as wheel: + for name, data in payload.items(): + wheel.writestr(name, data) + with zipfile.ZipFile(wheels / f"mssql_python_odbc-18.6.2.1-py3-none-{arch}.whl", "w") as wheel: + wheel.writestr("mssql_python_odbc/__init__.py", "") + env = dict( + os.environ, + PREFIX=str(prefix), + PYTHON=str(tmp_path / "nonexecutable-python") if cross_build else sys.executable, + PKG_NAME="mssql-python", + PKG_VERSION="1.13.0", + CONDA_PY=tag, + target_platform="win-arm64" if arch == "win_arm64" else "win-64", + WHEELS_DIR=str(wheels), + MSSQL_ODBC_VERSION="18.6.2.1", + PIP_TARGET=str(site_packages), + PIP_CONFIG_FILE=os.devnull, + PIP_USER="0", + ) + result = subprocess.run( + [ + os.environ["COMSPEC"], + "/d", + "/c", + str(_MODULE_PATH.parents[2] / "conda/mssql-python/bld.bat"), + ], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + output = result.stdout + result.stderr + if state in ("valid", "abi3"): + assert result.returncode == 0, output + assert (site_packages / "mssql_python_odbc/__init__.py").is_file() + else: + assert result.returncode != 0, output + assert "ERROR: required mssql_py_core" in output + + +def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312",)): """Build a minimal .conda (info-*.tar.zst + pkg-*.tar.zst) with the given payload files.""" name = "mssql-python-1.13.0-py312_0" @@ -186,7 +262,7 @@ def _make_conda(tmp_path, subdir, payload, depends=None): "version": "1.13.0", "build": "py312_0", "subdir": subdir, - "depends": depends if depends is not None else ["python_abi 3.12.* *_cp312"], + "depends": depends, } idx = json.dumps(index).encode() info_buf = io.BytesIO() @@ -202,6 +278,12 @@ def _make_conda(tmp_path, subdir, payload, depends=None): return str(conda_path) +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_malformed_dependencies_are_reported(tmp_path): + errors = ape.audit_package(_make_conda(tmp_path, "win-arm64", {}, depends=None)) + assert any("malformed" in error and "depends" in error for error in errors) + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") def test_win_arm64_arm64_binaries_pass(tmp_path): p = _make_conda( diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index 9e33a781..cf53844f 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -10,8 +10,12 @@ import importlib.util import io import json +import os +import shutil import struct +import subprocess import sys +import sysconfig import tarfile import zipfile from pathlib import Path @@ -133,7 +137,7 @@ def _zstd_compress(raw: bytes) -> bytes: return zstandard.ZstdCompressor().compress(raw) -def _make_conda(tmp_path, subdir, payload): +def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312",)): """Build a minimal .conda (info-*.tar.zst + pkg-*.tar.zst) with the given payload files.""" name = "mssql-python-1.13.0-py312_0" @@ -145,7 +149,7 @@ def _make_conda(tmp_path, subdir, payload): tf.addfile(ti, io.BytesIO(data)) index = {"name": "mssql-python", "version": "1.13.0", "build": "py312_0", "subdir": subdir} - index["depends"] = ["python_abi 3.12.* *_cp312"] + index["depends"] = depends idx = json.dumps(index).encode() info_buf = io.BytesIO() with tarfile.open(fileobj=info_buf, mode="w") as tf: @@ -160,6 +164,12 @@ def _make_conda(tmp_path, subdir, payload): return str(conda_path) +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_malformed_dependencies_are_reported(tmp_path): + errors = mac.audit_package(_make_conda(tmp_path, "osx-arm64", {}, depends=None)) + assert any("malformed" in error and "depends" in error for error in errors) + + _BINDING = "lib/python3.12/site-packages/mssql_python/ddbc_bindings.cp312-darwin.so" _CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-darwin.so" _CORE_INIT = "lib/python3.12/site-packages/mssql_py_core/__init__.py" @@ -187,6 +197,90 @@ def _realistic_payload(binding, arm64=None, x86_64=None): return payload +@pytest.mark.parametrize("cross_build", [False, True]) +@pytest.mark.parametrize("state", ["valid", "missing", "missing-init", "wrong-tag", "abi3"]) +def test_unix_recipe_requires_core_on_both_install_paths(tmp_path, cross_build, state): + bash = shutil.which("bash") + if not bash: + pytest.skip("Direct Unix recipe execution requires bash") + tools = subprocess.run([bash, "-c", "command -v unzip"], capture_output=True, timeout=10) + if tools.returncode: + pytest.skip("Direct Unix recipe execution requires unzip") + payload = _realistic_payload(_fake_macho_fat([_X86_64, _ARM64])) + core = _CORE + if not cross_build: + core = _CORE.rsplit("/", 1)[0] + "/mssql_py_core" + sysconfig.get_config_var("EXT_SUFFIX") + payload[core] = payload.pop(_CORE) + if state == "missing": + del payload[core] + elif state == "missing-init": + del payload[_CORE_INIT] + elif state == "wrong-tag": + payload[core + ".wrong-tag"] = payload.pop(core) + elif state == "abi3": + payload[_CORE.replace("cpython-312-darwin", "abi3")] = payload.pop(core) + wheels = tmp_path / "wheels" + wheels.mkdir() + code_tag = "cp312-cp312-macosx_15_0_universal2" if cross_build else "py3-none-any" + odbc_tag = "py3-none-macosx_15_0_universal2" if cross_build else "py3-none-any" + with zipfile.ZipFile(wheels / f"mssql_python-1.13.0-{code_tag}.whl", "w") as wheel: + for name, data in payload.items(): + if "/mssql_python_odbc/" not in name: + wheel.writestr(name.removeprefix("lib/python3.12/site-packages/"), data) + wheel.writestr( + "mssql_python-1.13.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: mssql-python\nVersion: 1.13.0\n", + ) + wheel.writestr( + "mssql_python-1.13.0.dist-info/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {code_tag}\n", + ) + wheel.writestr("mssql_python-1.13.0.dist-info/RECORD", "") + with zipfile.ZipFile(wheels / f"mssql_python_odbc-18.6.2.1-{odbc_tag}.whl", "w") as wheel: + wheel.writestr("mssql_python_odbc/__init__.py", "") + wheel.writestr( + "mssql_python_odbc-18.6.2.1.dist-info/METADATA", + "Metadata-Version: 2.1\nName: mssql-python-odbc\nVersion: 18.6.2.1\n", + ) + wheel.writestr( + "mssql_python_odbc-18.6.2.1.dist-info/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {odbc_tag}\n", + ) + wheel.writestr("mssql_python_odbc-18.6.2.1.dist-info/RECORD", "") + site_packages = tmp_path / "site-packages" + result = subprocess.run( + [bash, (_MODULE_PATH.parents[2] / "conda/mssql-python/build.sh").as_posix()], + env=dict( + os.environ, + PYTHON=( + (tmp_path / "nonexecutable-python").as_posix() + if cross_build + else Path(sys.executable).as_posix() + ), + PKG_NAME="mssql-python", + PKG_VERSION="1.13.0", + CONDA_PY="312", + WHEELS_DIR=wheels.as_posix(), + SP_DIR=site_packages.as_posix(), + PREFIX=(tmp_path / "prefix").as_posix(), + MSSQL_ODBC_VERSION="18.6.2.1", + PIP_TARGET=str(site_packages), + PIP_CONFIG_FILE=os.devnull, + PIP_USER="0", + ), + capture_output=True, + text=True, + timeout=30, + ) + output = result.stdout + result.stderr + if state in ("valid", "abi3"): + assert result.returncode == 0, output + assert (site_packages / "mssql_python_odbc/__init__.py").is_file() + else: + assert result.returncode != 0, output + assert "ERROR: required mssql_py_core" in output + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") @pytest.mark.parametrize("subdir", ["osx-arm64", "osx-64"]) def test_osx_packages_accept_real_split_driver_layout(tmp_path, subdir): From d217622a258c54b5ded2fac78a52a3bb9a8e627b Mon Sep 17 00:00:00 2001 From: Jahnvi Thakkar Date: Fri, 11 Sep 2026 17:03:29 +0530 Subject: [PATCH 12/12] DOC: Clarify unattended Conda channel terms prerequisite Document approved agent provisioning before defaults-backed Windows ARM64 solves. Keep automatic terms acceptance disabled and preserve explicit Conda failure diagnostics. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- conda/README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/conda/README.md b/conda/README.md index 02329d34..df46182b 100644 --- a/conda/README.md +++ b/conda/README.md @@ -37,8 +37,16 @@ load do not establish those results. Applicable OS, certificate and authenticati configuration remain external prerequisites. Use only organizationally approved channels and handle applicable terms separately; -the Windows ARM64 dependency profile includes Anaconda `defaults`. Run this existing -build workflow only in a disposable isolated installation: shared-environment +the Windows ARM64 dependency profile includes Anaconda `defaults`. Before unattended +Windows ARM64 builds, agent owners must use approved provisioning to put a disposable +Conda installation on `PATH`, with applicable channel terms handled for the job's +execution identity. The orchestrator's automatic Miniforge installation does not +establish that approval. If Conda enforces terms that have not been handled, the build +or verification solve stops with Conda's diagnostic; provision the prerequisite before +running again. Automatic acceptance, including an inherited +`CONDA_PLUGINS_AUTO_ACCEPT_TOS` opt-in, remains disabled. + +Run this existing build workflow only in a disposable isolated installation: shared-environment ownership hardening is outside this change. Publication/provenance tooling is proposed in a separate release-additions PR, without a required merge order; see the [release status and qualification caveats](../README.md#installation).