diff --git a/OneBranchPipelines/scripts/build_conda_packages.py b/OneBranchPipelines/scripts/build_conda_packages.py index 45a3d63b..945761a7 100644 --- a/OneBranchPipelines/scripts/build_conda_packages.py +++ b/OneBranchPipelines/scripts/build_conda_packages.py @@ -4,7 +4,7 @@ Replaces build-conda-packages.ps1 + build-conda-packages.sh (the same 7-step pipeline written twice, which had already drifted). conda is Python and every agent has a bootstrap interpreter, so ONE orchestrator runs on every leg; the platform differences (the Miniforge -installer, the win-arm64 Terms-of-Service auto-accept, the Linux-only reachability gate) are +installer, the win-arm64 channel profile, the Linux-only reachability gate) are a handful of branches, not a second 360-line script. Running as a NORMAL process also means the caller reads the exit code directly -- so the PowerShell ErrorActionPreference flips, the `2>$null` swallows, and the `cmd /c "exit 0"` reset all disappear. @@ -297,6 +297,7 @@ def build_env( ) -> dict[str, str]: """The environment consumed by the recipe (jinja + build.sh/bld.bat) and by conda-build.""" env = dict(os.environ) + env.pop("CONDA_PLUGINS_AUTO_ACCEPT_TOS", None) env["WHEELS_DIR"] = links env["MSSQL_PYTHON_VERSION"] = mssql_ver env["MSSQL_ODBC_VERSION"] = odbc_ver @@ -306,12 +307,6 @@ def build_env( # host can execute it (natively / Rosetta 2 / QEMU binfmt). env["CONDA_SUBDIR"] = cross_target_subdir _log(f"Cross-targeting conda subdir: CONDA_SUBDIR={cross_target_subdir}") - if cross_target_subdir == "win-arm64": - # win-arm64 deps (python 3.12-3.14, cryptography, vc14_runtime, pyodbc) live on - # Anaconda `defaults`, not conda-forge. Auto-accept the defaults ToS so the - # unattended host-env + verify solves never block on a prompt. - env["CONDA_PLUGINS_AUTO_ACCEPT_TOS"] = "yes" - _log("win-arm64: CONDA_PLUGINS_AUTO_ACCEPT_TOS=yes") if cross_target_subdir.endswith("aarch64") and os.path.isdir("/usr/aarch64-linux-gnu"): # Emulated aarch64 verify runs under qemu-user; point it at the aarch64 glibc loader. env.setdefault("QEMU_LD_PREFIX", "/usr/aarch64-linux-gnu") @@ -392,17 +387,16 @@ def audit_packages( env=env, what="RUNPATH self-containment audit", ) - # win-arm64 is cross-built on x64 where its runtime import is skipped, so its arch is - # trusted from the wheel filename UNLESS the PE machine assert reads it out of the payload. - if target_subdir == "win-arm64": + # Both Windows packages must retain the core; cross builds also rely on static architecture. + if target_subdir in ("win-64", "win-arm64"): pe = os.path.join(eng, "assert_pe_machine.py") if not os.path.isfile(pe): _die(f"PE machine-type assert script not found at {pe}") - _log("=== win-arm64 PE machine-type assert (vendored .pyd/.dll must be ARM64) ===") + _log(f"=== {target_subdir} PE machine-type and required native-component assert ===") run( - [conda, "run", "-n", builder, "python", pe, "--root", bld, "--subdir", "win-arm64"], + [conda, "run", "-n", builder, "python", pe, "--root", bld, "--subdir", target_subdir], env=env, - what="win-arm64 PE machine-type assert", + what=f"{target_subdir} PE machine-type assert", ) # osx legs: verify the universal binding contains the target slice and each thin vendored # driver dylib matches its architecture-specific directory. osx-arm64 is cross-built on the @@ -470,14 +464,27 @@ def _import_probe(mod_name: str, ok_label: str) -> str: INSTALLED package, not the checkout. Uses abspath (NOT realpath) so conda's softlink install mode -- where the site-packages entry symlinks into the pkgs/ cache OUTSIDE the prefix -- is not false-failed: the import PATH stays under the prefix regardless of hard/soft link; only - the symlink TARGET would not. Then prints ok_label + the version.""" + the symlink TARGET would not. Then prints ok_label + the installed module path.""" return ( f"import os,sys,{mod_name} as m;" "f=os.path.normcase(os.path.abspath(m.__file__));" "pref=os.path.normcase(os.path.abspath(sys.prefix));" f"assert f.startswith(pref+os.sep),{mod_name!r}+' loaded from '+m.__file__+" "', not under the conda env '+sys.prefix+' (stray PYTHONPATH/.pth?)';" - f"print({ok_label!r},m.__version__)" + f"print({ok_label!r},m.__file__)" + ) + + +def _core_probe() -> str: + return ( + _import_probe("mssql_py_core", "CORE_PACKAGE_OK") + ";import importlib.machinery;" + "exts=[v for k,v in list(sys.modules.items()) " + "if (k=='mssql_py_core' or k.startswith('mssql_py_core.')) " + "and isinstance(getattr(v,'__loader__',None),importlib.machinery.ExtensionFileLoader)];" + "assert exts,'mssql_py_core did not load its required native extension';" + "assert all(os.path.normcase(os.path.abspath(v.__file__)).startswith(pref+os.sep) " + "for v in exts),'core native extension loaded outside installed prefix';" + "print('CORE_NATIVE_OK',*[v.__file__ for v in exts])" ) @@ -610,6 +617,12 @@ def _verify_impl( f"arm64 cross-build. Refusing to silently skip validation. Output: {out}" ) + # A separate process prevents API/driver preloads from masking core load failures. + run( + [conda, "run", "-n", name, "python", "-c", _core_probe()], + env=env, + what=f"independent required mssql_py_core load (py {py})", + ) _log(f"=== [py {py}] import mssql_python + prove the vendored ODBC payload is present ===") run( [ diff --git a/README.md b/README.md index d24937dc..5638fd89 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The driver is compatible with all the Python versions >= 3.10 > **Important Note:** > > ### ODBC Driver Distribution -> The ODBC driver binaries used by `mssql-python` are distributed exclusively through a dedicated companion package: +> For **pip/PyPI installations**, the ODBC driver binaries used by `mssql-python` are distributed through a dedicated companion distribution: > > - Package: `mssql-python-odbc` > - Import name: `mssql_python_odbc` @@ -20,12 +20,18 @@ The driver is compatible with all the Python versions >= 3.10 > > `mssql-python` depends on `mssql-python-odbc==18.6.2.1`. The ODBC driver is loaded lazily when the first connection is created. `pip install mssql-python` transparently pulls the companion package alongside it — no separate install step is required. > -> Starting with v1.13.0, the bundled `libs/` fallback that shipped in v1.12.0 has been removed. Creating a connection will fail if `mssql-python-odbc` is not installed. If you install `mssql-python` from a private index or with `--no-deps`, make sure `mssql-python-odbc==18.6.2.1` is installed alongside it. +> Starting with v1.13.0, the bundled `libs/` fallback that shipped in v1.12.0 has been removed. For pip installations, creating a connection will fail if `mssql-python-odbc` is not installed. If you install `mssql-python` from a private index or with `--no-deps`, make sure `mssql-python-odbc==18.6.2.1` is installed alongside it. +> +> The **temporary Conda candidate** instead combines the code and ODBC payload in one `mssql-python` Conda package. It does not require a separately installed Conda ODBC package. This is not an announcement of public channel availability or release qualification; see the [Conda installation, migration, and readiness guide](conda/README.md). > > ### ODBC Provider Selection (opt-in) > `mssql-python` also supports selecting an alternate native ODBC provider before the first connection, via the `mssql_python.native_provider` module property or the `MSSQL_PYTHON_NATIVE_PROVIDER` environment variable (which takes precedence). A conflicting property assignment emits a `RuntimeWarning`. The default, `"msodbcsql18"`, is unchanged; opting into `"mssql-odbc"` requires the `mssql-python-rs` package (which bundles the Rust ODBC driver alongside the Rust TDS core). Call `mssql_python.get_native_provider_info()` to check the selected provider, source, package version, and resolved driver path. ## Installation + +The pip commands below describe the public release. The temporary combined Conda +candidate has a separate platform matrix and Linux compatibility floor; it is not +covered by the public release's production-readiness statement above. **Windows:** mssql-python can be installed with [pip](http://pypi.python.org/pypi/pip) ```bash @@ -60,6 +66,30 @@ tdnf distro-sync && tdnf install -y libtool-ltdl krb5-libs glibc-iconv pip install mssql-python ``` +**Conda candidate:** Obtain the exact candidate channel and version from its owner; +these changes do not publish packages to the public `microsoft` channel. The candidate +includes the ODBC Driver 18 payload and required bulk-copy core. Linux requires +**glibc >=2.34** for that complete payload; `krb5`, OpenSSL, and `libltdl` resolve from +`conda-forge`, so the system package steps above are not required. Windows uses SChannel. +On macOS, encrypted connections still require system OpenSSL from Homebrew +(`brew install openssl`) or MacPorts, not Conda OpenSSL. Windows ARM64 dependencies +resolve from `defaults`; handle channel terms separately, without automatic acceptance. +Use a fresh environment and replace the placeholders with the owner-provided values: +```bash +# Windows x64, macOS, and Linux +conda install -c "" -c microsoft -c conda-forge --strict-channel-priority --override-channels "mssql-python=" + +# Windows ARM64 +conda install -c "" -c microsoft -c defaults --override-channels "mssql-python=" +``` + +**Conda release status:** Publication tooling and its administrator prerequisites are +proposed separately in the [release-additions PR](https://github.com/microsoft/mssql-python/pull/720). +These native-packaging changes do not publish packages or require a particular merge order. +Static audits and import checks do not certify SQL, certificate-verified TLS, authentication, +bulk copy, or optional features across the full matrix. Production publication remains gated +on separate release controls and qualification; validate-only success does not authorize it. + ## Key Features ### Supported Platforms diff --git a/conda/README.md b/conda/README.md new file mode 100644 index 00000000..df46182b --- /dev/null +++ b/conda/README.md @@ -0,0 +1,54 @@ +# Combined Conda candidate + +This recipe combines the matching code and ODBC wheels into **one `mssql-python` +Conda package**, including the required bulk-copy core. Pip instead installs +`mssql-python-odbc` as a separate companion distribution. Neither requires a +separately installed ODBC driver or driver manager. + +Direct recipe builds must set `MSSQL_PYTHON_VERSION` to the exact selected code-wheel +version before rendering/building. The shared orchestrator derives and supplies it +automatically; omitted input fails recipe rendering instead of choosing a release. +Both native installation and cross extraction require the bulk-copy initializer and +a compatible extension filename. The separate native audits still validate binary headers. + +This is a temporary candidate, not an announcement of public channel availability. +Obtain the exact candidate archive/channel from its owner and install into a new +Conda environment. Activate it and select the same interpreter/kernel in your IDE +or notebook. Application imports and the public API remain unchanged. Use Conda +for upgrades; do not overwrite Conda-owned driver files with pip. + +Linux requires **glibc >=2.34** for the complete native payload, including the +bulk-copy core, even when the binding wheel has a lower platform tag. Do not force +installation on older glibc. This does not change the separate PyPI support claim. +macOS retains its external Homebrew/MacPorts OpenSSL prerequisite for encryption; +Conda OpenSSL alone does not satisfy the driver's system-path lookup. + +The release goal is the matching PyPI release's public API and supported feature +behavior on all 28 ordinary CPython variants: 3.10-3.14 on win-64, linux-64, +linux-aarch64, osx-64 and osx-arm64; 3.12-3.14 on win-arm64. No silent removal of +required native functionality is acceptable. Optional features require their +corresponding dependencies; Windows ARM64 PyArrow availability remains a blocker +to qualifying those features, not permission to drop them. + +These guards are **not full-matrix parity certification**. Native target execution, +SQL, certificate-verified TLS, authentication, bulk-copy and optional-feature tests +remain required before release. Cross-build/static checks and a DB-less driver +load do not establish those results. Applicable OS, certificate and authentication +configuration remain external prerequisites. + +Use only organizationally approved channels and handle applicable terms separately; +the Windows ARM64 dependency profile includes Anaconda `defaults`. Before unattended +Windows ARM64 builds, agent owners must use approved provisioning to put a disposable +Conda installation on `PATH`, with applicable channel terms handled for the job's +execution identity. The orchestrator's automatic Miniforge installation does not +establish that approval. If Conda enforces terms that have not been handled, the build +or verification solve stops with Conda's diagnostic; provision the prerequisite before +running again. Automatic acceptance, including an inherited +`CONDA_PLUGINS_AUTO_ACCEPT_TOS` opt-in, remains disabled. + +Run this existing build workflow only in a disposable isolated installation: shared-environment +ownership hardening is outside this change. Publication/provenance tooling is proposed +in a separate release-additions PR, without a required merge order; see the +[release status and qualification caveats](../README.md#installation). +Neither this native-packaging change nor validate-only success authorizes production +publication. diff --git a/conda/mssql-python/bld.bat b/conda/mssql-python/bld.bat index 6e9991fd..c0a41653 100644 --- a/conda/mssql-python/bld.bat +++ b/conda/mssql-python/bld.bat @@ -39,23 +39,22 @@ if errorlevel 1 ( echo ERROR: extracted "!CODE_WHL!" has no mssql_python\ddbc_bindings.cp%CONDA_PY% pyd ^(wrong-Python binding^). exit /b 1 ) - REM Keep mssql_py_core when the wheel provides a matching-arch native ext so bulk copy - REM ships (PR #737 makes the win-arm64 wheel vendor the arm64 core). If only the legacy - REM x64 core is present (a pre-#737 wheel), strip it so the package never carries a core - REM that can't load on the target -- the .pyd name encodes the arch. Bulk copy then lazily - REM reports "not available"; the rest of the DBAPI works. Mirrors the ddbc check above. - if exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" ( - echo Keeping matching-arch mssql_py_core; bulk copy enabled on the !ODBC_ARCH! package. - ) else ( - echo No cp%CONDA_PY%-!ODBC_ARCH! mssql_py_core in the wheel; removing the mismatched core ^(bulk copy unavailable until the arm64-core wheel ships^). - if exist "%SP%\mssql_py_core" rmdir /s /q "%SP%\mssql_py_core" - if exist "%SP%\mssql_py_core.libs" rmdir /s /q "%SP%\mssql_py_core.libs" - ) ) else ( "%PYTHON%" -m pip install --no-deps --no-index --find-links "%WHEELS_DIR%" %PKG_NAME%==%PKG_VERSION% -vv if errorlevel 1 exit /b 1 ) +REM Both install paths require bulk copy. The PE audit still checks actual architecture. +if not exist "%SP%\mssql_py_core\__init__.py" ( + echo ERROR: required mssql_py_core initializer is missing. Use a corrected upstream wheel; refusing reduced functionality. + exit /b 1 +) +REM Bare .pyd is the Windows stable-ABI suffix. +if not exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" if not exist "%SP%\mssql_py_core\mssql_py_core.pyd" ( + echo ERROR: required mssql_py_core is missing or incompatible with cp%CONDA_PY% !ODBC_ARCH!. Use a corrected upstream wheel; refusing reduced functionality. + exit /b 1 +) + REM Extract the arch-specific odbc wheel into the SAME site-packages so REM mssql_python_odbc\libs\ sits beside mssql_python\ and the loader finds the driver. REM The py3-none tag only means "no Python bytecode" -- the vendored driver DLLs ARE diff --git a/conda/mssql-python/build.sh b/conda/mssql-python/build.sh index 355b1a6b..66b5adb8 100644 --- a/conda/mssql-python/build.sh +++ b/conda/mssql-python/build.sh @@ -18,7 +18,9 @@ odbc_ver="${MSSQL_ODBC_VERSION:?MSSQL_ODBC_VERSION not set}" if "$PYTHON" -c "import sys" >/dev/null 2>&1; then "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "$PKG_NAME==$PKG_VERSION" -vv "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "mssql-python-odbc==$odbc_ver" -vv + core_suffix="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))')" else + core_suffix=".cpython-${CONDA_PY}-darwin.so" echo "Host Python '$PYTHON' is not executable on this agent (non-emulated cross-build);" echo "extracting both wheels into \$SP_DIR without running Python." mkdir -p "$SP_DIR" @@ -26,7 +28,9 @@ else # universal2 wheels are cpXY-specific (compiled ddbc_bindings), so filter on the # target CONDA_PY to never grab another interpreter's wheel (mirrors bld.bat). code_whl="" - for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-cp${CONDA_PY}-"*.whl; do + # Match macosx explicitly (like the odbc glob below) so a stray Linux cpXY wheel staged in + # the same dir can never be picked up on this macOS-only cross branch. + for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-cp${CONDA_PY}-"*macosx*.whl; do [ -e "$w" ] && { code_whl="$w"; break; } done [ -n "$code_whl" ] || { echo "ERROR: no ${PKG_NAME}==${PKG_VERSION} cp${CONDA_PY} wheel in '$WHEELS_DIR'" >&2; exit 1; } @@ -52,6 +56,15 @@ else unzip -oq "$odbc_whl" -d "$SP_DIR" fi +# Both install paths require bulk copy; the platform audits still check binary headers. +if [ ! -f "$SP_DIR/mssql_py_core/__init__.py" ] || { + [ ! -f "$SP_DIR/mssql_py_core/mssql_py_core${core_suffix}" ] && + [ ! -f "$SP_DIR/mssql_py_core/mssql_py_core.abi3.so" ] +}; then + echo "ERROR: required mssql_py_core initializer or compatible extension is missing. Use a corrected upstream wheel; refusing reduced functionality." >&2 + exit 1 +fi + # --------------------------------------------------------------------------- # Linux driver reachability (#563) -- the core fix. # --------------------------------------------------------------------------- diff --git a/conda/mssql-python/meta.yaml b/conda/mssql-python/meta.yaml index cbc39fdd..fb02fb7f 100644 --- a/conda/mssql-python/meta.yaml +++ b/conda/mssql-python/meta.yaml @@ -1,4 +1,4 @@ -{% set version = environ.get('MSSQL_PYTHON_VERSION', '1.14.0') %} +{% set version = environ['MSSQL_PYTHON_VERSION'] %} package: name: mssql-python @@ -45,6 +45,8 @@ requirements: - python # conda does NOT inherit the wheel's install_requires, so pin azure-identity here. - azure-identity >=1.12.0 + # Defaults' Windows ARM64 Python 3.12 does not export its CPython ABI constraint. + - python_abi 3.12.* *_cp312 # [win and arm64 and py == 312] # ODBC Driver 18 payload deps (the driver ships inside this package, so its # security-serviced deps are declared here). OpenSSL for TLS is dlopen'd, so # overlinking can't see it; pinned <4 (Driver 18 supports the OpenSSL 1.1/3.0 ABI @@ -63,8 +65,8 @@ requirements: # msodbcsql18.dll imports VCRUNTIME140.dll but the vendored vcredist ships only # msvcp140.dll; declare the serviced conda runtime. - vc14_runtime # [win] - # Re-assert the wheel's platform floor (conda drops the wheel tag); never stricter. - - __glibc >=2.28 # [linux] + # The complete payload, including the required bulk-copy core, needs GLIBC_2.34. + - __glibc >=2.34 # [linux] - __osx >=15.0 # [osx] test: diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 373f8dd0..00497f09 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -154,6 +154,9 @@ jobs: sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" -C + # Pool eviction tests observe physical connections from a second session. + sqlcmd -S "localhost" -U "sa" -P "$env:DB_PASSWORD" -b -Q "GRANT VIEW SERVER PERFORMANCE STATE TO [testuser]; EXECUTE AS LOGIN = 'testuser'; SELECT connection_id FROM sys.dm_exec_connections WHERE session_id = @@SPID; REVERT;" -C + if ($LASTEXITCODE -ne 0) { throw "Failed to provision the CI test login's DMV-read permission." } displayName: 'Setup database and user for SQL Server 2022' condition: eq(variables['sqlVersion'], 'SQL2022') env: @@ -220,6 +223,9 @@ jobs: sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" -C sqlcmd -S "localhost" -U "sa" -P "$(DB_PASSWORD)" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" -C + # Pool eviction tests observe physical connections from a second session. + sqlcmd -S "localhost" -U "sa" -P "$env:DB_PASSWORD" -b -Q "GRANT VIEW SERVER PERFORMANCE STATE TO [testuser]; EXECUTE AS LOGIN = 'testuser'; SELECT connection_id FROM sys.dm_exec_connections WHERE session_id = @@SPID; REVERT;" -C + if ($LASTEXITCODE -ne 0) { throw "Failed to provision the CI test login's DMV-read permission." } displayName: 'Setup database and user for SQL Server 2025' condition: eq(variables['sqlVersion'], 'SQL2025') env: diff --git a/eng/scripts/_conda_pkg.py b/eng/scripts/_conda_pkg.py index cfc87cdb..8bed56d1 100644 --- a/eng/scripts/_conda_pkg.py +++ b/eng/scripts/_conda_pkg.py @@ -14,9 +14,79 @@ import io import json +import re import tarfile import zipfile -from typing import Any, Iterator +from typing import Any, Iterable, Iterator + + +def validate_native_contract( + members: Iterable[tuple[str, bytes]], index: dict[str, Any] +) -> list[str]: + """Require a target binding, core extension and initializer; platform audits check headers. + + Wheels may include bindings for several Python minors. The core uses Python's + normal extension loader, including its stable-ABI suffix. These static checks + do not replace native import/feature qualification. + """ + pins = [ + d + for d in index.get("depends", []) + if isinstance(d, str) and d.split()[:1] == ["python_abi"] + ] + abi = re.fullmatch(r"python_abi (3\.\d+)\.\* \*_cp(3\d+)", pins[0]) if len(pins) == 1 else None + if abi is None or abi[1].replace(".", "") != abi[2]: + return ["expected a matching normal CPython python_abi pin"] + subdir = index["subdir"] + windows = subdir.startswith("win-") + suffix = "pyd" if windows else "so" + prefix = "Lib" if windows else f"lib/python{abi[1]}" + root = f"{prefix}/site-packages/" + names = [name.replace("\\", "/") for name, _ in members] + bindings = [ + name + for name in names + if re.fullmatch( + rf"{re.escape(root)}mssql_python/ddbc_bindings\.cp{abi[2]}-[^/]+\.{suffix}", name + ) + ] + cores = [ + name + for name in names + if re.fullmatch( + rf"{re.escape(root)}mssql_py_core/mssql_py_core(?:\.[^/]+)?\.{suffix}", name + ) + ] + arch = { + "win-64": "win_amd64", + "win-arm64": "win_arm64", + "linux-64": "x86_64-linux-gnu", + "linux-aarch64": "aarch64-linux-gnu", + "osx-64": "darwin", + "osx-arm64": "darwin", + }[subdir] + core_names = ( + (f"mssql_py_core.cp{abi[2]}-{arch}.pyd", "mssql_py_core.pyd") + if windows + else (f"mssql_py_core.cpython-{abi[2]}-{arch}.so", "mssql_py_core.abi3.so") + ) + errors = [] + initializer = f"{root}mssql_py_core/__init__.py" + if names.count(initializer) != 1: + errors.append( + f"expected exactly one required {initializer}; found {names.count(initializer)}" + ) + if len(bindings) != 1: + errors.append( + f"expected exactly one normal cp{abi[2]} native binding; found {len(bindings)}" + ) + if len(cores) != 1: + errors.append( + f"expected exactly one required mssql_py_core native extension; found {len(cores)}" + ) + elif cores[0].rsplit("/", 1)[-1] not in core_names: + errors.append(f"{cores[0]}: mssql_py_core is incompatible with normal cp{abi[2]} {subdir}") + return errors def zstd_decompress(raw: bytes) -> bytes: @@ -87,11 +157,18 @@ def read_index(path: str) -> dict[str, Any]: member = tf.extractfile("info/index.json") if member is None: raise ValueError("info/index.json missing") - return json.load(member) - if path.endswith(".tar.bz2"): + index = json.load(member) + elif path.endswith(".tar.bz2"): with tarfile.open(path, "r:bz2") as tf: member = tf.extractfile("info/index.json") if member is None: raise ValueError("info/index.json missing") - return json.load(member) - raise ValueError("unrecognized conda package extension") + index = json.load(member) + else: + raise ValueError("unrecognized conda package extension") + if not isinstance(index, dict): + raise ValueError("info/index.json must be an object") + depends = index.get("depends", []) + if not isinstance(depends, list) or any(not isinstance(dep, str) for dep in depends): + raise ValueError("info/index.json depends must be a list of dependency strings") + return index diff --git a/eng/scripts/assert_macho_arch.py b/eng/scripts/assert_macho_arch.py index d4afe6ed..77ba03a6 100644 --- a/eng/scripts/assert_macho_arch.py +++ b/eng/scripts/assert_macho_arch.py @@ -28,7 +28,11 @@ import struct import sys -from _conda_pkg import iter_payload_members as _iter_payload_members, read_index +from _conda_pkg import ( + iter_payload_members as _iter_payload_members, + read_index, + validate_native_contract, +) # Mach-O cputype (mach/machine.h): the base type OR'd with the 64-bit ABI flag -> lipo name. _CPU_ARCH_ABI64 = 0x01000000 @@ -152,7 +156,8 @@ def audit_package(path: str) -> list[str]: """Return violation strings for one package (empty == clean / skipped non-macOS).""" base_name = os.path.basename(path) try: - subdir = read_subdir(path) + index = read_index(path) + subdir = str(index.get("subdir", "")) except Exception as exc: # malformed must FAIL, never silently skip return [f"{base_name}: unreadable/malformed package metadata ({exc})."] @@ -166,7 +171,7 @@ def audit_package(path: str) -> list[str]: except ValueError as exc: # malformed payload (e.g. .conda missing pkg-*.tar.zst) return [f"{base_name}: unreadable/malformed package payload ({exc})."] - errors: list[str] = [] + errors = validate_native_contract(members, index) binding_seen = 0 target_driver_libraries: set[str] = set() for name, data in members: @@ -178,6 +183,8 @@ def audit_package(path: str) -> list[str]: if "/mssql_python/" in low and base_low.startswith("ddbc_bindings") and low.endswith(".so"): binding_seen += 1 required_arch = expected + elif "/mssql_py_core/" in low and low.endswith(".so"): + required_arch = expected elif "/mssql_python_odbc/libs/macos/" in low and low.endswith(".dylib"): relative = low.split("/mssql_python_odbc/libs/macos/", 1)[1] parts = relative.split("/") diff --git a/eng/scripts/assert_pe_machine.py b/eng/scripts/assert_pe_machine.py index ae339a5e..05cc0a27 100644 --- a/eng/scripts/assert_pe_machine.py +++ b/eng/scripts/assert_pe_machine.py @@ -24,7 +24,11 @@ import struct import sys -from _conda_pkg import iter_payload_members as _iter_payload_members, read_index +from _conda_pkg import ( + iter_payload_members as _iter_payload_members, + read_index, + validate_native_contract, +) # IMAGE_FILE_MACHINE_* (winnt.h): the PE COFF Machine field -> a short name. _MACHINES = { @@ -86,7 +90,8 @@ def audit_package(path: str) -> list[str]: """Return violation strings for one package (empty == clean / skipped non-Windows).""" base_name = os.path.basename(path) try: - subdir = read_subdir(path) + index = read_index(path) + subdir = str(index.get("subdir", "")) except Exception as exc: # malformed must FAIL, never silently skip return [f"{base_name}: unreadable/malformed package metadata ({exc})."] @@ -101,7 +106,7 @@ def audit_package(path: str) -> list[str]: except ValueError as exc: # malformed payload (e.g. .conda missing pkg-*.tar.zst) return [f"{base_name}: unreadable/malformed package payload ({exc})."] - errors: list[str] = [] + errors = validate_native_contract(members, index) native_seen = 0 binding_seen = 0 driver_dll_seen = 0 diff --git a/eng/scripts/audit_bundled_binaries.py b/eng/scripts/audit_bundled_binaries.py index a4afa81d..f7698e9b 100644 --- a/eng/scripts/audit_bundled_binaries.py +++ b/eng/scripts/audit_bundled_binaries.py @@ -44,17 +44,25 @@ import glob import os import posixpath +import re import struct import sys from typing import Any, Iterable, TypedDict -from _conda_pkg import iter_payload_members as _iter_payload_members, read_index +from _conda_pkg import ( + iter_payload_members as _iter_payload_members, + read_index, + validate_native_contract, +) # --- ELF constants --------------------------------------------------------- _DT_NEEDED = 1 _DT_STRTAB = 5 +_DT_STRSZ = 10 _DT_RPATH = 15 _DT_RUNPATH = 29 +_DT_VERNEED = 0x6FFFFFFE +_DT_VERNEEDNUM = 0x6FFFFFFF _PT_LOAD = 1 _PT_DYNAMIC = 2 @@ -104,6 +112,7 @@ class _ElfDynamicInfo(TypedDict): runpath: str | None rpath: str | None needed: list[str] + glibc_required: list[tuple[int, ...]] def _is_elf(data: bytes) -> bool: @@ -130,9 +139,11 @@ def elf_dynamic(data: bytes) -> _ElfDynamicInfo: table that a stripped/rewritten binary might not carry. Handles ELF32/ELF64 and both endiannesses; the shipped drivers are ELF64-LE. """ - out: _ElfDynamicInfo = {"runpath": None, "rpath": None, "needed": []} + out: _ElfDynamicInfo = {"runpath": None, "rpath": None, "needed": [], "glibc_required": []} if not _is_elf(data): - return out + raise ValueError("not a complete ELF header") + if data[4] not in (1, 2) or data[5] not in (1, 2): + raise ValueError("invalid ELF class or endianness") is64 = data[4] == 2 en = "<" if data[5] == 1 else ">" @@ -144,15 +155,20 @@ def elf_dynamic(data: bytes) -> _ElfDynamicInfo: e_phoff = struct.unpack_from(en + "I", data, 0x1C)[0] e_phentsize = struct.unpack_from(en + "H", data, 0x2A)[0] e_phnum = struct.unpack_from(en + "H", data, 0x2C)[0] - if not e_phoff or not e_phnum: - return out + if ( + not e_phoff + or not e_phnum + or e_phentsize < (56 if is64 else 32) + or e_phoff + e_phnum * e_phentsize > len(data) + ): + raise ValueError("invalid or truncated ELF program headers") loads = [] # (p_vaddr, p_offset, p_filesz) dyn = None # (p_offset, p_filesz) for i in range(e_phnum): off = e_phoff + i * e_phentsize if off + e_phentsize > len(data): - return out + raise ValueError("truncated ELF program header") p_type = struct.unpack_from(en + "I", data, off)[0] if is64: p_offset = struct.unpack_from(en + "Q", data, off + 8)[0] @@ -162,25 +178,33 @@ def elf_dynamic(data: bytes) -> _ElfDynamicInfo: p_offset = struct.unpack_from(en + "I", data, off + 4)[0] p_vaddr = struct.unpack_from(en + "I", data, off + 8)[0] p_filesz = struct.unpack_from(en + "I", data, off + 16)[0] + if p_offset + p_filesz > len(data): + raise ValueError("ELF segment extends beyond the file") if p_type == _PT_LOAD: loads.append((p_vaddr, p_offset, p_filesz)) elif p_type == _PT_DYNAMIC: dyn = (p_offset, p_filesz) if dyn is None: - return out + raise ValueError("ELF has no PT_DYNAMIC segment") dyn_off, dyn_size = dyn - def vaddr_to_off(vaddr: int) -> int | None: + def vaddr_to_off(vaddr: int, size: int = 1) -> int: for v, o, sz in loads: - if v <= vaddr < v + sz: + if v <= vaddr and vaddr + size <= v + sz: return vaddr - v + o - return None + raise ValueError("ELF dynamic address is outside a file-backed PT_LOAD segment") strtab_vaddr = None + strtab_size = None + verneed_vaddr = None + verneed_num = None runpath_rel = None rpath_rel = None needed_rel: list[int] = [] entsize = 16 if is64 else 8 + terminated = False + if dyn_size % entsize: + raise ValueError("ELF dynamic segment has a partial entry") for off in range(dyn_off, dyn_off + dyn_size, entsize): if off + entsize > len(data): break @@ -191,6 +215,7 @@ def vaddr_to_off(vaddr: int) -> int | None: d_tag = struct.unpack_from(en + "i", data, off)[0] d_val = struct.unpack_from(en + "I", data, off + 4)[0] if d_tag == 0: # DT_NULL terminates the array + terminated = True break if d_tag == _DT_STRTAB: strtab_vaddr = d_val @@ -200,25 +225,78 @@ def vaddr_to_off(vaddr: int) -> int | None: rpath_rel = d_val elif d_tag == _DT_NEEDED: needed_rel.append(d_val) - if strtab_vaddr is None: - return out - strtab_off = vaddr_to_off(strtab_vaddr) - if strtab_off is None: - return out + elif d_tag == _DT_STRSZ: + strtab_size = d_val + elif d_tag == _DT_VERNEED: + verneed_vaddr = d_val + elif d_tag == _DT_VERNEEDNUM: + verneed_num = d_val + if not terminated or strtab_vaddr is None or not strtab_size: + raise ValueError("ELF dynamic segment lacks DT_NULL, DT_STRTAB or DT_STRSZ") + strtab_off = vaddr_to_off(strtab_vaddr, strtab_size) def read_str(rel: int) -> str: + if not 0 <= rel < strtab_size: + raise ValueError("ELF string offset is outside DT_STRTAB") pos = strtab_off + rel - end = data.find(b"\x00", pos) - return data[pos : (end if end >= 0 else len(data))].decode("utf-8", "replace") + end = data.find(b"\x00", pos, strtab_off + strtab_size) + if end < 0: + raise ValueError("unterminated ELF dynamic string") + return data[pos:end].decode("utf-8", "strict") if runpath_rel is not None: out["runpath"] = read_str(runpath_rel) if rpath_rel is not None: out["rpath"] = read_str(rpath_rel) out["needed"] = [read_str(n) for n in needed_rel] + if (verneed_vaddr is None) != (verneed_num is None): + raise ValueError("ELF version requirements need both DT_VERNEED and DT_VERNEEDNUM") + if verneed_vaddr is not None: + if not verneed_num or verneed_num > len(data) // 16: + raise ValueError("invalid ELF version requirement count") + current = verneed_vaddr + for number in range(verneed_num): + offset = vaddr_to_off(current, 16) + version, count, library, aux, next_need = struct.unpack_from(en + "HHIII", data, offset) + if version != 1 or not count or count > len(data) // 16 or aux < 16: + raise ValueError("invalid ELF version requirement record") + read_str(library) + auxiliary = current + aux + for item in range(count): + offset = vaddr_to_off(auxiliary, 16) + _, _, _, name, next_aux = struct.unpack_from(en + "IHHII", data, offset) + requirement = read_str(name) + if requirement.startswith("GLIBC_"): + match = re.fullmatch(r"GLIBC_(\d+(?:\.\d+)+)", requirement) + if match is None: + raise ValueError(f"unsupported glibc symbol requirement {requirement}") + out["glibc_required"].append(tuple(map(int, match[1].split(".")))) + if item < count - 1 and next_aux < 16: + raise ValueError("truncated ELF version auxiliary chain") + if item == count - 1 and next_aux != 0: + raise ValueError("ELF version auxiliary count disagrees with chain") + auxiliary += next_aux + if number < verneed_num - 1 and next_need < 16: + raise ValueError("truncated ELF version requirement chain") + if number == verneed_num - 1 and next_need != 0: + raise ValueError("ELF version requirement count disagrees with chain") + current += next_need return out +def declared_glibc_floor(index: dict[str, Any]) -> tuple[int, ...]: + """Require a single explicit minimum; wheel platform tags are not symbol-floor evidence.""" + specs = [ + d for d in index.get("depends", []) if isinstance(d, str) and d.split()[:1] == ["__glibc"] + ] + if len(specs) != 1: + raise ValueError("expected exactly one __glibc >=VERSION dependency") + match = re.fullmatch(r"__glibc\s+>=(\d+(?:\.\d+)+)", specs[0]) + if match is None: + raise ValueError(f"unsupported __glibc dependency: {specs[0]!r}") + return tuple(map(int, match[1].split("."))) + + def effective_runpath(dyn: _ElfDynamicInfo) -> str | None: """The loader ignores ``DT_RPATH`` when ``DT_RUNPATH`` is present.""" return dyn["runpath"] if dyn["runpath"] is not None else dyn["rpath"] @@ -342,6 +420,13 @@ def audit_package(path: str) -> list[str]: except ValueError as exc: # malformed payload (e.g. .conda missing pkg-*.tar.zst) return [f"{base_name}: unreadable/malformed package payload ({exc})."] + errors.extend(validate_native_contract(members, index)) + try: + glibc_floor = declared_glibc_floor(index) + except (TypeError, ValueError) as exc: + errors.append(f"{base_name}: invalid glibc compatibility metadata: {exc}") + glibc_floor = None + for name, data in members: base = posixpath.basename(name) norm = "/" + name @@ -364,11 +449,15 @@ def audit_package(path: str) -> list[str]: is_driver = any(base.startswith(p) for p in _DRIVER_PREFIXES) is_inst = base == _ODBCINST - if not (is_driver or is_inst): + is_native = data.startswith(b"\x7fELF") or base.endswith(".so") or ".so." in base + if not (is_native or is_driver or is_inst): continue if not _is_elf(data): errors.append(f"{name}: expected an ELF binary but the header is not ELF.") continue + if data[4:6] != b"\x02\x01": + errors.append(f"{name}: expected an ELF64 little-endian binary for '{subdir}'.") + continue # Architecture gate: the ELF machine MUST match the package's conda subdir, so # an x86_64 driver mislabeled under a linux-aarch64 package (which the emulated @@ -379,10 +468,26 @@ def audit_package(path: str) -> list[str]: errors.append( f"{name}: ELF machine {mach} ({machine_name}) does " f"not match the '{subdir}' package arch {expected_machine} " - f"({_MACHINE_NAME[expected_machine]}) -- wrong-arch/mislabeled driver." + f"({_MACHINE_NAME[expected_machine]}) -- wrong-arch/mislabeled native binary." ) - dyn = elf_dynamic(data) + try: + dyn = elf_dynamic(data) + except (ValueError, struct.error) as exc: + errors.append(f"{name}: invalid ELF dynamic metadata ({exc}).") + continue + for required in dyn["glibc_required"]: + if glibc_floor is not None: + width = max(len(required), len(glibc_floor)) + if required + (0,) * (width - len(required)) > glibc_floor + (0,) * ( + width - len(glibc_floor) + ): + errors.append( + f"{name}: requires GLIBC_{'.'.join(map(str, required))} but archive " + f"declares __glibc >={'.'.join(map(str, glibc_floor))}." + ) + if not (is_driver or is_inst): + continue raw_runpath = effective_runpath(dyn) entries = _entries(raw_runpath) needed = dyn["needed"] diff --git a/setup.py b/setup.py index 70c8aae4..bb43e269 100644 --- a/setup.py +++ b/setup.py @@ -199,6 +199,8 @@ def run(self): "mssql_py_core": [ "mssql_py_core.cp*.pyd", "mssql_py_core.cp*.so", + "mssql_py_core.pyd", + "mssql_py_core.abi3.so", ], } diff --git a/tests/test_029_bundled_binary_audit.py b/tests/test_029_bundled_binary_audit.py index b516cc0c..c096e9ba 100644 --- a/tests/test_029_bundled_binary_audit.py +++ b/tests/test_029_bundled_binary_audit.py @@ -53,7 +53,7 @@ def _load_module(): audit = _load_module() -def _make_elf64(runpath=None, rpath=None, needed=(), machine=62): +def _make_elf64(runpath=None, rpath=None, needed=(), machine=62, versions=()): """Build a minimal, self-consistent ELF64-LE with real program headers. Emits a PT_LOAD (vaddr == file offset, covering the whole file) + a PT_DYNAMIC, @@ -78,8 +78,17 @@ def add_str(s): rp_rel = add_str(runpath) if runpath is not None else None rpath_rel = add_str(rpath) if rpath is not None else None need_rels = [add_str(n) for n in needed] - - dynamic_off = dynstr_off + len(dynstr) + version_rels = [add_str(n) for n in versions] + library_rel = add_str("libc.so.6") if versions else 0 + verneed_off = dynstr_off + len(dynstr) + verneed = b"" + if versions: + verneed = struct.pack("=3,<4"] +_GOOD_DEPENDS = [ + "python >=3.12,<3.13.0a0", + "python_abi 3.12.* *_cp312", + "__glibc >=2.34", + "azure-identity", + "krb5", + "libtool", + "openssl >=3,<4", +] +_BINDING = "lib/python3.12/site-packages/mssql_python/ddbc_bindings.cp312-x86_64.so" +_CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-x86_64-linux-gnu.so" +_CORE_INIT = "lib/python3.12/site-packages/mssql_py_core/__init__.py" _DISTROS_BY_SUBDIR = { "linux-64": ("alpine", "debian_ubuntu", "rhel", "suse"), "linux-aarch64": ("alpine", "debian_ubuntu", "rhel"), @@ -146,11 +170,12 @@ def _make_pkg( rpath=None, subdir="linux-64", vendored=None, - depends=None, + depends=tuple(_GOOD_DEPENDS), driver_needed=None, inst_needed=None, machine=62, distros=None, + native_payload=None, ): """Write a minimal package with complete per-distro driver trees by default.""" p = tmp_path / "mssql-python-1.13.0-py312_0.tar.bz2" @@ -169,11 +194,22 @@ def add(name, data): "version": "1.13.0", "build": "py312_0", "subdir": subdir, - "depends": _GOOD_DEPENDS if depends is None else depends, + "depends": depends, } ).encode(), ) arch = "arm64" if subdir == "linux-aarch64" else "x86_64" + extension_arch = "aarch64" if subdir == "linux-aarch64" else "x86_64" + if native_payload is None: + native_payload = { + _CORE_INIT: b"from .mssql_py_core import *\n", + _BINDING.replace("x86_64", extension_arch): _make_elf64(machine=machine), + _CORE.replace("x86_64", extension_arch): _make_elf64( + machine=machine, versions=("GLIBC_2.2.5", "GLIBC_2.34") + ), + } + for name, data in native_payload.items(): + add(name, data) selected_distros = distros or _DISTROS_BY_SUBDIR.get(subdir, ("debian_ubuntu",)) for distro in selected_distros: libdir = ( @@ -205,6 +241,15 @@ def add(name, data): # --- low-level parser ------------------------------------------------------- +@pytest.mark.parametrize( + "depends", + [None, 17, "python_abi 3.12.* *_cp312", {"python_abi": "3.12"}, ["python_abi", None]], +) +def test_audit_reports_malformed_dependency_field(tmp_path, depends): + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("malformed" in error and "depends" in error for error in errors) + + def test_elf_dynamic_pt_parse(): data = _make_elf64(runpath=_GOOD_RUNPATH, needed=["libkrb5.so.3", "libodbcinst.so.2"]) dyn = audit.elf_dynamic(data) @@ -234,6 +279,118 @@ def test_audit_passes_with_exact_climb(tmp_path): assert audit.audit_package(_make_pkg(tmp_path)) == [] +@pytest.mark.parametrize("missing", [_BINDING, _CORE, _CORE_INIT]) +def test_full_feature_package_requires_binding_and_core(tmp_path, missing): + native = {_BINDING: _make_elf64(), _CORE: _make_elf64(), _CORE_INIT: b""} + del native[missing] + errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + assert any("exactly one" in error for error in errors) + + +@pytest.mark.parametrize("component", [_BINDING, _CORE]) +def test_every_required_extension_checks_elf_arch_not_filename(tmp_path, component): + native = {_BINDING: _make_elf64(), _CORE: _make_elf64(), _CORE_INIT: b""} + native[component] = _make_elf64(machine=183) + errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + assert any(component in error and "does not match" in error for error in errors) + + +@pytest.mark.parametrize("component", [_BINDING, _CORE]) +@pytest.mark.parametrize("wrong_tag", ["311", "312t", "312d"]) +def test_required_extensions_reject_wrong_or_non_normal_abi(tmp_path, component, wrong_tag): + native = {_BINDING: _make_elf64(), _CORE: _make_elf64(), _CORE_INIT: b""} + data = native.pop(component) + native[component.replace("312", wrong_tag)] = data + assert audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + + +def test_core_abi3_and_extra_normal_bindings_are_supported(tmp_path): + native = { + _CORE_INIT: b"from .mssql_py_core import *\n", + _BINDING: _make_elf64(), + _BINDING.replace("312", "310"): _make_elf64(), + _CORE.replace("cpython-312-x86_64-linux-gnu", "abi3"): _make_elf64(), + } + assert audit.audit_package(_make_pkg(tmp_path, native_payload=native)) == [] + + +@pytest.mark.parametrize( + "abi", + [ + "python_abi 3.12.* *_cp312t", + "python_abi 3.12.* *_cp311", + "python_abi 3.11.* *_cp311", + "python_abi >=3.12", + ], +) +def test_archive_requires_consistent_normal_python_abi_metadata(tmp_path, abi): + depends = [abi if d.startswith("python_abi ") else d for d in _GOOD_DEPENDS] + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("python_abi" in error or "native binding" in error for error in errors) + + +@pytest.mark.parametrize("declared", ["2.28", "2.33"]) +def test_core_symbol_floor_cannot_exceed_archive_glibc_minimum(tmp_path, declared): + depends = [f"__glibc >={declared}" if d.startswith("__glibc ") else d for d in _GOOD_DEPENDS] + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any(_CORE in error and "GLIBC_2.34" in error for error in errors) + + +@pytest.mark.parametrize("declared", ["2.34", "2.34.0", "2.35"]) +def test_core_symbol_floor_compatible_with_archive_minimum(tmp_path, declared): + depends = [f"__glibc >={declared}" if d.startswith("__glibc ") else d for d in _GOOD_DEPENDS] + assert audit.audit_package(_make_pkg(tmp_path, depends=depends)) == [] + + +def test_symbol_floor_is_read_from_version_needs_not_arbitrary_bytes(tmp_path): + native = { + _CORE_INIT: b"from .mssql_py_core import *\n", + _BINDING: _make_elf64(), + _CORE: _make_elf64(versions=("GLIBC_2.34",)) + b"GLIBC_99.99\x00", + } + assert audit.audit_package(_make_pkg(tmp_path, native_payload=native)) == [] + + +def test_auxiliary_native_library_symbol_floor_is_checked(tmp_path): + extra = "lib/python3.12/site-packages/mssql_py_core.libs/libsupport.so.1" + native = { + _CORE_INIT: b"from .mssql_py_core import *\n", + _BINDING: _make_elf64(), + _CORE: _make_elf64(), + extra: _make_elf64(versions=("GLIBC_2.35",)), + } + errors = audit.audit_package(_make_pkg(tmp_path, native_payload=native)) + assert any(extra in error and "GLIBC_2.35" in error for error in errors) + + +@pytest.mark.parametrize("spec", [None, "__glibc >=2.28|>=2.34", "__glibc >=2.34junk"]) +def test_missing_or_ambiguous_glibc_floor_is_rejected(tmp_path, spec): + depends = [d for d in _GOOD_DEPENDS if not d.startswith("__glibc")] + if spec: + depends.append(spec) + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("glibc compatibility metadata" in error for error in errors) + + +@pytest.mark.parametrize("damage", ["truncated", "version-address", "aux-chain", "class"]) +def test_malformed_required_core_elf_cannot_pass(tmp_path, damage): + core = bytearray(_make_elf64(versions=("GLIBC_2.2.5", "GLIBC_2.34"))) + if damage == "truncated": + core = core[:70] + elif damage == "version-address": + tag = core.index(struct.pack(" bytes: @@ -129,7 +134,119 @@ def test_zstd_backend_is_available_for_conda_audit_tests(): ) -def _make_conda(tmp_path, subdir, payload): +def test_wheel_retains_normal_and_stable_abi_core_extensions(tmp_path): + pytest.importorskip("setuptools", reason="Wheel archive regression requires setuptools") + pytest.importorskip("wheel", reason="Wheel archive regression requires the wheel build backend") + shutil.copy2(_MODULE_PATH.parents[2] / "setup.py", tmp_path / "setup.py") + sources = { + "PyPI_Description.md": "Packaging fixture", + "mssql_python/__init__.py": "", + "mssql_python_odbc/__init__.py": '__version__ = "18.6.2.1"\n', + "mssql_py_core/__init__.py": "from .mssql_py_core import *\n", + } + for relative, content in sources.items(): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + extensions = ( + "mssql_py_core.cp312-win_arm64.pyd", + "mssql_py_core.cpython-312-x86_64-linux-gnu.so", + "mssql_py_core.pyd", + "mssql_py_core.abi3.so", + ) + for name in extensions: + (tmp_path / "mssql_py_core" / name).write_bytes(b"native payload fixture") + + result = subprocess.run( + [sys.executable, "setup.py", "--quiet", "bdist_wheel"], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stdout + result.stderr + wheels = list((tmp_path / "dist").glob("*.whl")) + assert len(wheels) == 1 + with zipfile.ZipFile(wheels[0]) as wheel: + for name in extensions: + assert wheel.read(f"mssql_py_core/{name}") == b"native payload fixture" + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows recipe requires cmd.exe") +@pytest.mark.parametrize("cross_build", [False, True]) +@pytest.mark.parametrize("state", ["valid", "missing", "missing-init", "wrong-tag", "abi3"]) +def test_windows_recipe_requires_core_on_both_install_paths(tmp_path, cross_build, state): + wheels = tmp_path / "wheels" + wheels.mkdir() + prefix = tmp_path / "prefix" + site_packages = prefix / "Lib" / "site-packages" + tag = f"{sys.version_info.major}{sys.version_info.minor}" + arch = sysconfig.get_platform().replace("-", "_") + machine = _ARM64 if arch == "win_arm64" else _AMD64 + core = f"mssql_py_core/mssql_py_core.cp{tag}-{arch}.pyd" + payload = { + "mssql_python/__init__.py": b"", + f"mssql_python/ddbc_bindings.cp{tag}-{arch}.pyd": _fake_pe(machine), + "mssql_py_core/__init__.py": b"from .mssql_py_core import *\n", + core: _fake_pe(machine), + } + if state == "missing": + del payload[core] + elif state == "missing-init": + del payload["mssql_py_core/__init__.py"] + elif state == "wrong-tag": + payload[core.replace(f".cp{tag}-", ".cp999-")] = payload.pop(core) + elif state == "abi3": + payload["mssql_py_core/mssql_py_core.pyd"] = payload.pop(core) + dist_info = "mssql_python-1.13.0.dist-info" + payload[f"{dist_info}/METADATA"] = ( + b"Metadata-Version: 2.1\nName: mssql-python\nVersion: 1.13.0\n" + ) + payload[f"{dist_info}/WHEEL"] = ( + f"Wheel-Version: 1.0\nRoot-Is-Purelib: false\nTag: cp{tag}-cp{tag}-{arch}\n".encode() + ) + payload[f"{dist_info}/RECORD"] = "\n".join(f"{name},," for name in payload).encode() + with zipfile.ZipFile(wheels / f"mssql_python-1.13.0-cp{tag}-cp{tag}-{arch}.whl", "w") as wheel: + for name, data in payload.items(): + wheel.writestr(name, data) + with zipfile.ZipFile(wheels / f"mssql_python_odbc-18.6.2.1-py3-none-{arch}.whl", "w") as wheel: + wheel.writestr("mssql_python_odbc/__init__.py", "") + env = dict( + os.environ, + PREFIX=str(prefix), + PYTHON=str(tmp_path / "nonexecutable-python") if cross_build else sys.executable, + PKG_NAME="mssql-python", + PKG_VERSION="1.13.0", + CONDA_PY=tag, + target_platform="win-arm64" if arch == "win_arm64" else "win-64", + WHEELS_DIR=str(wheels), + MSSQL_ODBC_VERSION="18.6.2.1", + PIP_TARGET=str(site_packages), + PIP_CONFIG_FILE=os.devnull, + PIP_USER="0", + ) + result = subprocess.run( + [ + os.environ["COMSPEC"], + "/d", + "/c", + str(_MODULE_PATH.parents[2] / "conda/mssql-python/bld.bat"), + ], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + output = result.stdout + result.stderr + if state in ("valid", "abi3"): + assert result.returncode == 0, output + assert (site_packages / "mssql_python_odbc/__init__.py").is_file() + else: + assert result.returncode != 0, output + assert "ERROR: required mssql_py_core" in output + + +def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312",)): """Build a minimal .conda (info-*.tar.zst + pkg-*.tar.zst) with the given payload files.""" name = "mssql-python-1.13.0-py312_0" @@ -145,6 +262,7 @@ def _make_conda(tmp_path, subdir, payload): "version": "1.13.0", "build": "py312_0", "subdir": subdir, + "depends": depends, } idx = json.dumps(index).encode() info_buf = io.BytesIO() @@ -160,13 +278,21 @@ def _make_conda(tmp_path, subdir, payload): return str(conda_path) +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_malformed_dependencies_are_reported(tmp_path): + errors = ape.audit_package(_make_conda(tmp_path, "win-arm64", {}, depends=None)) + assert any("malformed" in error and "depends" in error for error in errors) + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") def test_win_arm64_arm64_binaries_pass(tmp_path): p = _make_conda( tmp_path, "win-arm64", { + _CORE_INIT: b"from .mssql_py_core import *\n", "Lib/site-packages/mssql_python/ddbc_bindings.cp312-arm64.pyd": _fake_pe(_ARM64), + "Lib/site-packages/mssql_py_core/mssql_py_core.cp312-win_arm64.pyd": _fake_pe(_ARM64), "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/msodbcsql18.dll": _fake_pe( _ARM64 ), @@ -178,6 +304,52 @@ def test_win_arm64_arm64_binaries_pass(tmp_path): assert ape.audit_package(p) == [] +@pytest.mark.parametrize( + "state", + [ + "valid", + "missing", + "missing-init", + "wrong-arch", + "wrong-tag", + "abi3", + "missing-abi", + "wrong-abi", + ], +) +def test_required_core_contract(tmp_path, state): + pin = "python_abi 3.12.* *_cp312" + # The observed defaults CP312 host supplied only the Python range, not an ABI export. + depends = ["vc14_runtime", "python >=3.12,<3.13.0a0", "azure-identity >=1.12.0"] + if state != "missing-abi": + depends.append(pin if state != "wrong-abi" else "python_abi 3.12.* *_cp313") + core = "Lib/site-packages/mssql_py_core/mssql_py_core.cp312-win_arm64.pyd" + payload = { + _CORE_INIT: b"from .mssql_py_core import *\n", + "Lib/site-packages/mssql_python/ddbc_bindings.cp312-arm64.pyd": _fake_pe(_ARM64), + core: _fake_pe(_ARM64), + "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/msodbcsql18.dll": _fake_pe(_ARM64), + "Lib/site-packages/mssql_python_odbc/libs/windows/arm64/mssql-auth.dll": _fake_pe(_ARM64), + } + if state == "missing": + del payload[core] + elif state == "missing-init": + del payload[_CORE_INIT] + elif state == "wrong-arch": + payload[core] = _fake_pe(_AMD64) + elif state == "wrong-tag": + payload[core.replace("312", "311")] = payload.pop(core) + elif state == "abi3": + payload[core.replace(".cp312-win_arm64", "")] = payload.pop(core) + errors = ape.audit_package(_make_conda(tmp_path, "win-arm64", payload, depends=depends)) + if state in ("valid", "abi3"): + assert errors == [] + elif state in ("missing-abi", "wrong-abi"): + assert any("matching normal CPython python_abi pin" in error for error in errors) + else: + assert any("mssql_py_core" in error for error in errors) + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") def test_win_arm64_driver_dlls_in_x64_directory_fail_presence_gate(tmp_path): # Correct ARM64 machine fields do not help when the loader searches windows/arm64, diff --git a/tests/test_034_conda_verify_cwd.py b/tests/test_034_conda_verify_cwd.py index 413be486..a00d5b0e 100644 --- a/tests/test_034_conda_verify_cwd.py +++ b/tests/test_034_conda_verify_cwd.py @@ -29,8 +29,6 @@ / "scripts" / "build_conda_packages.py" ) -_PIPELINE_PATH = _ORCH_PATH.parent.parent / "conda-build-pipeline.yml" -_CONSOLIDATE_JOB_PATH = _ORCH_PATH.parent.parent / "jobs" / "consolidate-conda-artifacts-job.yml" pytestmark = pytest.mark.skipif( not _ORCH_PATH.exists(), reason=f"orchestrator not present ({_ORCH_PATH})" @@ -45,58 +43,6 @@ def _load_orchestrator(): return mod -def test_best_effort_consolidation_runs_after_upstream_failure(): - pipeline = _PIPELINE_PATH.read_text(encoding="utf-8") - for stage_name in ("CondaWin64", "CondaMacOS", "CondaLinux"): - producer = pipeline.split(f"- stage: {stage_name}", 1)[1] - assert "dependsOn: ValidateWheelProvenance" in producer.split("jobs:", 1)[0] - - stage = pipeline.split("- stage: ConsolidateConda", 1)[1] - dependencies = stage.split("jobs:", 1)[0] - for stage_name in ("CondaWin64", "CondaMacOS", "CondaLinux"): - assert f"- {stage_name}" in dependencies - assert "condition: succeededOrFailed()" in stage.split("jobs:", 1)[0] - - mac_stage = pipeline.split("- stage: CondaMacOS", 1)[1].split("- stage: CondaLinux", 1)[0] - mac_publish = mac_stage.split("displayName: 'Publish macOS conda artifact'", 1)[1] - assert "condition: succeededOrFailed()" in mac_publish.split("inputs:", 1)[0] - - job = _CONSOLIDATE_JOB_PATH.read_text(encoding="utf-8") - consolidate = job.split("- job: ConsolidateArtifacts", 1)[1] - assert "condition: succeededOrFailed()" in consolidate.split("pool:", 1)[0] - - -def test_official_builds_require_main_wheel_provenance(): - pipeline = _PIPELINE_PATH.read_text(encoding="utf-8") - resource = pipeline.split("- pipeline: buildPipeline", 1)[1].split("extends:", 1)[0] - assert "branch: main" in resource - - gate = pipeline.split("- stage: ValidateWheelProvenance", 1)[1].split("- stage: CondaWin64", 1)[ - 0 - ] - assert '[[ -z "${WHEEL_SOURCE_BRANCH:-}" ]]' in gate - assert 'case "$ONEBRANCH_TYPE" in' in gate - assert "Official)" in gate - assert "NonOfficial) ;;" in gate - assert '[[ "$WHEEL_SOURCE_BRANCH" != "refs/heads/main" ]]' in gate - assert "unknown OneBranch type" in gate - assert "ONEBRANCH_TYPE: ${{ variables.effectiveOneBranchType }}" in gate - assert "WHEEL_SOURCE_BRANCH: $(resources.pipeline.buildPipeline.sourceBranch)" in gate - - for stage_name in ("CondaWin64", "CondaMacOS", "CondaLinux"): - producer = pipeline.split(f"- stage: {stage_name}", 1)[1] - assert "dependsOn: ValidateWheelProvenance" in producer.split("jobs:", 1)[0] - - -def test_windows_pool_demand_is_indented_under_demands_key(): - pipeline = _PIPELINE_PATH.read_text(encoding="utf-8") - windows_stage = pipeline.split("- stage: CondaWin64", 1)[1].split("- stage: CondaMacOS", 1)[0] - assert ( - " demands:\n" - " - imageOverride -equals PYTHON-1ES-MMS2022\n" in windows_stage - ) - - @pytest.mark.parametrize( ("target_subdir", "cross_build", "host", "expected"), [ @@ -338,7 +284,7 @@ def _fake_run(cmd, *args, **kwargs): import_calls = [ (cmd, cwd) for cmd, cwd in calls - if "-c" in cmd and any("mssql_python" in str(a) for a in cmd) + if "-c" in cmd and any("mssql_python" in str(a) or "mssql_py_core" in str(a) for a in cmd) ] assert import_calls, "verify() never issued an `import mssql_python` probe" for cmd, cwd in import_calls: @@ -346,6 +292,10 @@ def _fake_run(cmd, *args, **kwargs): f"import probe ran from {cwd!r}, not the neutral workdir {str(workdir)!r} -- the " f"repo source tree would shadow the conda-installed package" ) + codes = [cmd[-1] for cmd, _ in import_calls] + assert codes[0] == mod._core_probe() + assert codes.count(mod._core_probe()) == 1 + assert "import mssql_python" not in codes[0] def test_verify_restores_cwd_when_the_phase_fails(tmp_path, monkeypatch): @@ -491,6 +441,20 @@ def test_build_env_sets_subdir_for_cross_build(monkeypatch): assert env["CONDA_SUBDIR"] == "osx-arm64" +def test_recipe_requires_explicit_wheel_version(): + jinja2 = pytest.importorskip("jinja2", reason="Conda recipe rendering requires Jinja2") + recipe = _ORCH_PATH.parents[2] / "conda" / "mssql-python" / "meta.yaml" + template = jinja2.Environment(undefined=jinja2.StrictUndefined).from_string( + recipe.read_text(encoding="utf-8") + ) + env = _load_orchestrator().build_env("1.2.3", "18.6.2", "wheels", "") + assert 'version: "1.2.3"' in template.render(environ=env) + + del env["MSSQL_PYTHON_VERSION"] + with pytest.raises(jinja2.UndefinedError, match="MSSQL_PYTHON_VERSION"): + template.render(environ=env) + + def test_win_arm64_real_environment_create_failure_is_blocking(tmp_path, monkeypatch): """A successful solve does not prove package extraction/linking succeeds.""" mod = _load_orchestrator() @@ -599,6 +563,7 @@ def test_gather_wheels_rejects_multiple_odbc_matches(tmp_path): ("target_subdir", "expected_channels"), [ ("", ["microsoft", "conda-forge"]), + ("win-64", ["microsoft", "conda-forge"]), ("osx-arm64", ["microsoft", "conda-forge"]), ("linux-aarch64", ["microsoft", "conda-forge"]), ("win-arm64", ["defaults", "microsoft", "conda-forge"]), @@ -636,3 +601,108 @@ def _capture_run(command, **_kwargs): assert [command[index + 1] for index, arg in enumerate(command) if arg == "-c"] == ( expected_channels ) + + +@pytest.mark.parametrize("state", ["native", "pure-python", "foreign"]) +def test_core_probe_requires_native_extension_from_installed_prefix(state, tmp_path, monkeypatch): + mod = _load_orchestrator() + prefix = tmp_path / "prefix" + monkeypatch.setattr(sys, "prefix", str(prefix)) + package = types.ModuleType("mssql_py_core") + package.__file__ = str(prefix / "mssql_py_core" / "__init__.py") + monkeypatch.setitem(sys.modules, "mssql_py_core", package) + if state != "pure-python": + native = types.ModuleType("mssql_py_core.mssql_py_core") + native.__file__ = str((tmp_path / "foreign" if state == "foreign" else prefix) / "core.pyd") + native.__loader__ = importlib.machinery.ExtensionFileLoader( + native.__name__, native.__file__ + ) + monkeypatch.setitem(sys.modules, native.__name__, native) + if state == "native": + exec(mod._core_probe(), {}) + else: + with pytest.raises(AssertionError, match="native extension|outside installed prefix"): + exec(mod._core_probe(), {}) + + +def test_core_failure_blocks_api_preload(tmp_path, monkeypatch): + mod = _load_orchestrator() + calls = [] + + def fake_run(cmd, **kwargs): + calls.append(list(cmd)) + return types.SimpleNamespace(returncode=17 if mod._core_probe() in cmd else 0, stdout="") + + monkeypatch.setattr( + mod, + "subprocess", + types.SimpleNamespace(run=fake_run, PIPE=subprocess.PIPE, STDOUT=subprocess.STDOUT), + ) + with pytest.raises(SystemExit): + mod._verify_impl( + "conda", "channel", str(tmp_path), ["3.12"], "1.14.0", "linux-64", False, {} + ) + assert any(mod._core_probe() in cmd for cmd in calls) + assert not any("BINDING_OK" in str(cmd) for cmd in calls) + + +@pytest.mark.parametrize("subdir", ["win-64", "win-arm64"]) +def test_both_windows_targets_run_native_audit(subdir, monkeypatch): + mod = _load_orchestrator() + calls = [] + monkeypatch.setattr(mod, "run", lambda cmd, **kwargs: calls.append(cmd)) + mod.audit_packages( + "conda", "builder", str(_ORCH_PATH.parents[2] / "conda"), "output", subdir, {} + ) + pe_calls = [ + cmd for cmd in calls if any(str(arg).endswith("assert_pe_machine.py") for arg in cmd) + ] + assert len(pe_calls) == 1 + assert pe_calls[0][-2:] == ["--subdir", subdir] + + +@pytest.mark.parametrize("inherited", [None, "true"]) +def test_build_does_not_automatically_accept_channel_terms(monkeypatch, inherited): + mod = _load_orchestrator() + if inherited is None: + monkeypatch.delenv("CONDA_PLUGINS_AUTO_ACCEPT_TOS", raising=False) + else: + monkeypatch.setenv("CONDA_PLUGINS_AUTO_ACCEPT_TOS", inherited) + assert "CONDA_PLUGINS_AUTO_ACCEPT_TOS" not in mod.build_env( + "1.14.0", "18.6.2.1", "wheels", "win-arm64" + ) + assert os.environ.get("CONDA_PLUGINS_AUTO_ACCEPT_TOS") == inherited + + +@pytest.mark.parametrize("subdir", ["win-64", "win-arm64"]) +def test_main_routes_effective_target_to_native_audit(subdir, tmp_path, monkeypatch): + mod = _load_orchestrator() + targets = [] + monkeypatch.setattr(mod, "gather_wheels", lambda *args: ("1.14.0", "18.6.2.1")) + monkeypatch.setattr(mod, "find_or_install_conda", lambda *args: "conda") + monkeypatch.setattr(mod, "create_builder_env", lambda *args: "builder") + monkeypatch.setattr(mod, "detect_pythons", lambda *args: ["3.12"]) + monkeypatch.setattr(mod, "make_verify_channel", lambda *args: "channel") + for name in ("run", "build_packages", "verify", "stage"): + monkeypatch.setattr(mod, name, lambda *args, **kwargs: None) + monkeypatch.setattr(mod, "audit_packages", lambda *args: targets.append(args[-2])) + args = [ + "--mssql-wheel-dir", + str(tmp_path / "wheels"), + "--odbc-wheel-dir", + str(tmp_path / "wheels"), + "--odbc-wheel-filter", + "*.whl", + "--recipe-root", + str(_ORCH_PATH.parents[2] / "conda"), + "--output-dir", + str(tmp_path / "out"), + "--stage-dir", + str(tmp_path / "stage"), + "--conda-subdir", + subdir, + ] + if subdir == "win-arm64": + args += ["--conda-target-subdir", subdir] + assert mod.main(args) == 0 + assert targets == [subdir] diff --git a/tests/test_035_conda_macho_assert.py b/tests/test_035_conda_macho_assert.py index 2eda5058..cf53844f 100644 --- a/tests/test_035_conda_macho_assert.py +++ b/tests/test_035_conda_macho_assert.py @@ -10,8 +10,12 @@ import importlib.util import io import json +import os +import shutil import struct +import subprocess import sys +import sysconfig import tarfile import zipfile from pathlib import Path @@ -133,7 +137,7 @@ def _zstd_compress(raw: bytes) -> bytes: return zstandard.ZstdCompressor().compress(raw) -def _make_conda(tmp_path, subdir, payload): +def _make_conda(tmp_path, subdir, payload, depends=("python_abi 3.12.* *_cp312",)): """Build a minimal .conda (info-*.tar.zst + pkg-*.tar.zst) with the given payload files.""" name = "mssql-python-1.13.0-py312_0" @@ -145,6 +149,7 @@ def _make_conda(tmp_path, subdir, payload): tf.addfile(ti, io.BytesIO(data)) index = {"name": "mssql-python", "version": "1.13.0", "build": "py312_0", "subdir": subdir} + index["depends"] = depends idx = json.dumps(index).encode() info_buf = io.BytesIO() with tarfile.open(fileobj=info_buf, mode="w") as tf: @@ -159,7 +164,15 @@ def _make_conda(tmp_path, subdir, payload): return str(conda_path) +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_malformed_dependencies_are_reported(tmp_path): + errors = mac.audit_package(_make_conda(tmp_path, "osx-arm64", {}, depends=None)) + assert any("malformed" in error and "depends" in error for error in errors) + + _BINDING = "lib/python3.12/site-packages/mssql_python/ddbc_bindings.cp312-darwin.so" +_CORE = "lib/python3.12/site-packages/mssql_py_core/mssql_py_core.cpython-312-darwin.so" +_CORE_INIT = "lib/python3.12/site-packages/mssql_py_core/__init__.py" _DRIVER_ROOT = "lib/python3.12/site-packages/mssql_python_odbc/libs/macos" _DRIVER_LIBRARIES = ( "libltdl.7.dylib", @@ -173,13 +186,101 @@ def _realistic_payload(binding, arm64=None, x86_64=None): """Mirror the wheel's two architecture-specific four-library driver directories.""" arm64 = arm64 or _fake_macho_thin(_ARM64) x86_64 = x86_64 or _fake_macho_thin(_X86_64) - payload = {_BINDING: binding} + payload = { + _BINDING: binding, + _CORE: _fake_macho_fat([_X86_64, _ARM64]), + _CORE_INIT: b"from .mssql_py_core import *\n", + } for library in _DRIVER_LIBRARIES: payload[f"{_DRIVER_ROOT}/arm64/lib/{library}"] = arm64 payload[f"{_DRIVER_ROOT}/x86_64/lib/{library}"] = x86_64 return payload +@pytest.mark.parametrize("cross_build", [False, True]) +@pytest.mark.parametrize("state", ["valid", "missing", "missing-init", "wrong-tag", "abi3"]) +def test_unix_recipe_requires_core_on_both_install_paths(tmp_path, cross_build, state): + bash = shutil.which("bash") + if not bash: + pytest.skip("Direct Unix recipe execution requires bash") + tools = subprocess.run([bash, "-c", "command -v unzip"], capture_output=True, timeout=10) + if tools.returncode: + pytest.skip("Direct Unix recipe execution requires unzip") + payload = _realistic_payload(_fake_macho_fat([_X86_64, _ARM64])) + core = _CORE + if not cross_build: + core = _CORE.rsplit("/", 1)[0] + "/mssql_py_core" + sysconfig.get_config_var("EXT_SUFFIX") + payload[core] = payload.pop(_CORE) + if state == "missing": + del payload[core] + elif state == "missing-init": + del payload[_CORE_INIT] + elif state == "wrong-tag": + payload[core + ".wrong-tag"] = payload.pop(core) + elif state == "abi3": + payload[_CORE.replace("cpython-312-darwin", "abi3")] = payload.pop(core) + wheels = tmp_path / "wheels" + wheels.mkdir() + code_tag = "cp312-cp312-macosx_15_0_universal2" if cross_build else "py3-none-any" + odbc_tag = "py3-none-macosx_15_0_universal2" if cross_build else "py3-none-any" + with zipfile.ZipFile(wheels / f"mssql_python-1.13.0-{code_tag}.whl", "w") as wheel: + for name, data in payload.items(): + if "/mssql_python_odbc/" not in name: + wheel.writestr(name.removeprefix("lib/python3.12/site-packages/"), data) + wheel.writestr( + "mssql_python-1.13.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: mssql-python\nVersion: 1.13.0\n", + ) + wheel.writestr( + "mssql_python-1.13.0.dist-info/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {code_tag}\n", + ) + wheel.writestr("mssql_python-1.13.0.dist-info/RECORD", "") + with zipfile.ZipFile(wheels / f"mssql_python_odbc-18.6.2.1-{odbc_tag}.whl", "w") as wheel: + wheel.writestr("mssql_python_odbc/__init__.py", "") + wheel.writestr( + "mssql_python_odbc-18.6.2.1.dist-info/METADATA", + "Metadata-Version: 2.1\nName: mssql-python-odbc\nVersion: 18.6.2.1\n", + ) + wheel.writestr( + "mssql_python_odbc-18.6.2.1.dist-info/WHEEL", + f"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: {odbc_tag}\n", + ) + wheel.writestr("mssql_python_odbc-18.6.2.1.dist-info/RECORD", "") + site_packages = tmp_path / "site-packages" + result = subprocess.run( + [bash, (_MODULE_PATH.parents[2] / "conda/mssql-python/build.sh").as_posix()], + env=dict( + os.environ, + PYTHON=( + (tmp_path / "nonexecutable-python").as_posix() + if cross_build + else Path(sys.executable).as_posix() + ), + PKG_NAME="mssql-python", + PKG_VERSION="1.13.0", + CONDA_PY="312", + WHEELS_DIR=wheels.as_posix(), + SP_DIR=site_packages.as_posix(), + PREFIX=(tmp_path / "prefix").as_posix(), + MSSQL_ODBC_VERSION="18.6.2.1", + PIP_TARGET=str(site_packages), + PIP_CONFIG_FILE=os.devnull, + PIP_USER="0", + ), + capture_output=True, + text=True, + timeout=30, + ) + output = result.stdout + result.stderr + if state in ("valid", "abi3"): + assert result.returncode == 0, output + assert (site_packages / "mssql_python_odbc/__init__.py").is_file() + else: + assert result.returncode != 0, output + assert "ERROR: required mssql_py_core" in output + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") @pytest.mark.parametrize("subdir", ["osx-arm64", "osx-64"]) def test_osx_packages_accept_real_split_driver_layout(tmp_path, subdir): @@ -192,6 +293,27 @@ def test_osx_packages_accept_real_split_driver_layout(tmp_path, subdir): assert mac.audit_package(p) == [] +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +@pytest.mark.parametrize("state", ["missing", "missing-init", "wrong-arch", "wrong-tag", "abi3"]) +def test_required_core_contract(tmp_path, state): + payload = _realistic_payload(_fake_macho_fat([_X86_64, _ARM64])) + if state == "missing": + del payload[_CORE] + elif state == "missing-init": + del payload[_CORE_INIT] + elif state == "wrong-arch": + payload[_CORE] = _fake_macho_thin(_X86_64) + elif state == "wrong-tag": + payload[_CORE.replace("312", "311")] = payload.pop(_CORE) + else: + payload[_CORE.replace("cpython-312-darwin", "abi3")] = payload.pop(_CORE) + errors = mac.audit_package(_make_conda(tmp_path, "osx-arm64", payload)) + if state == "abi3": + assert errors == [] + else: + assert any("mssql_py_core" in error for error in errors) + + @pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") @pytest.mark.parametrize("missing_library", _DRIVER_LIBRARIES) def test_target_driver_runtime_requires_every_library(tmp_path, missing_library): diff --git a/tests/test_release_build_dependencies.py b/tests/test_release_build_dependencies.py deleted file mode 100644 index a66bda64..00000000 --- a/tests/test_release_build_dependencies.py +++ /dev/null @@ -1,397 +0,0 @@ -import re -from pathlib import Path - -import pytest - -ROOT = Path(__file__).parents[1] -ENG = ROOT / "eng" -WORKFLOW = ROOT / ".github" / "workflows" / "refresh-build-dependencies.yml" - -if not ENG.is_dir() or not (ROOT / "OneBranchPipelines").is_dir(): - pytest.skip( - "release dependency contracts require a complete source checkout", - allow_module_level=True, - ) - -DIRECT_REQUIREMENTS = { - "requirements-build-linux": {"pip", "pybind11", "pytest", "setuptools", "wheel"}, - "requirements-build-macos": {"cmake", "cryptography", "pip", "wheel"}, - "requirements-build-odbc": {"build", "pip", "setuptools", "twine", "wheel"}, - "requirements-build-windows": { - "pip", - "psutil", - "pybind11", - "pyodbc", - "pytest", - "setuptools", - "wheel", - "zstandard", - }, -} - -PIPELINE_LOCKS = { - "OneBranchPipelines/stages/build-linux-single-stage.yml": { - "/workspace/eng/requirements-build-linux.txt": 2, - "/workspace/eng/requirements-test-linux.txt": 2, - }, - "OneBranchPipelines/stages/build-macos-single-stage.yml": { - "eng/requirements-build-macos.txt": 1, - }, - "OneBranchPipelines/stages/build-odbc-all-stage.yml": { - "eng/requirements-build-odbc.txt": 1, - }, - "OneBranchPipelines/stages/build-windows-single-stage.yml": { - "eng/requirements-build-windows.txt": 1, - }, -} - -SUPPORTED_PYTHONS = "3.10 3.11 3.12 3.13 3.14" -LINUX_PLATFORMS = ( - "x86_64-manylinux_2_28 aarch64-manylinux_2_28 " - "x86_64-unknown-linux-musl aarch64-unknown-linux-musl" -) - -VALIDATION_MATRIX = { - "Linux build": ( - "linux", - "requirements-build-linux.txt", - SUPPORTED_PYTHONS, - LINUX_PLATFORMS, - ), - "Linux test": ( - "linux", - "requirements-test-linux.txt", - SUPPORTED_PYTHONS, - LINUX_PLATFORMS, - ), - "macOS": ( - "macos", - "requirements-build-macos.txt", - SUPPORTED_PYTHONS, - "x86_64-apple-darwin aarch64-apple-darwin", - ), - "Windows build host": ( - "windows", - "requirements-build-windows.txt", - SUPPORTED_PYTHONS, - "x86_64-pc-windows-msvc", - ), - "ODBC": ( - "odbc", - "requirements-build-odbc.txt", - "3.12", - "x86_64-pc-windows-msvc", - ), -} - -PIN = re.compile( - r"(?P[A-Za-z0-9][A-Za-z0-9._-]*)==(?P[^;\s\\]+)" r"(?:\s*;\s*[^\\]+)?\s+\\" -) -HASH = re.compile(r"--hash=sha256:[0-9a-f]{64}(?:\s+\\)?") -ACTION = re.compile(r"^\s*-\s+uses:\s+([^@\s]+)@([^\s#]+)", re.MULTILINE) - - -def _canonicalize(name): - return re.sub(r"[-_.]+", "-", name).lower() - - -def _active_lines(path): - return [ - line.strip() - for line in path.read_text(encoding="utf-8").splitlines() - if line.strip() and not line.lstrip().startswith("#") - ] - - -def _requirement_names(lines): - return { - _canonicalize(re.match(r"[A-Za-z0-9][A-Za-z0-9._-]*", line).group()) - for line in lines - if not line.startswith(("-r ", "-c ")) - } - - -def _lock_versions(path): - text = path.read_text(encoding="utf-8") - assert "--index-url" not in text - assert "--trusted-host" not in text - - versions = {} - current_name = None - hashes = [] - continued = False - - def finish_entry(): - if current_name is None: - return - assert hashes - assert len(hashes) == len(set(hashes)) - assert not continued - - for line in text.splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - - if line[0].isspace(): - assert current_name is not None and continued - match = HASH.fullmatch(stripped) - assert match - hashes.append(stripped.removesuffix("\\").rstrip()) - continued = stripped.endswith("\\") - continue - - finish_entry() - match = PIN.fullmatch(line) - assert match - current_name = _canonicalize(match["name"]) - assert current_name not in versions - versions[current_name] = match["version"] - hashes = [] - continued = True - - finish_entry() - assert versions - return versions - - -def _section(text, start, end=None): - start_index = text.index(start) - end_index = text.index(end, start_index) if end else len(text) - return text[start_index:end_index] - - -def _matrix_item(section, name): - marker = f" - name: {name}\n" - return _section(section, marker).split("\n - name:", maxsplit=1)[0] - - -@pytest.fixture(scope="module") -def workflow(): - return WORKFLOW.read_text(encoding="utf-8") - - -@pytest.mark.parametrize("stem, expected", DIRECT_REQUIREMENTS.items()) -def test_platform_locks_cover_their_direct_requirements(stem, expected): - assert _requirement_names(_active_lines(ENG / f"{stem}.in")) == expected - assert expected <= _lock_versions(ENG / f"{stem}.txt").keys() - - -@pytest.mark.parametrize( - "lock_name", - [f"{stem}.txt" for stem in DIRECT_REQUIREMENTS] + ["requirements-test-linux.txt"], -) -def test_lockfiles_are_exactly_pinned_and_sha256_hashed(lock_name): - _lock_versions(ENG / lock_name) - - -@pytest.mark.parametrize( - "invalid_lock", - ( - "demo>=1\n", - "demo @ https://example.invalid/demo.whl\n", - f" --hash=sha256:{'a' * 64}\n", - f"demo==1 \\\n --hash=sha512:{'a' * 128}\n", - "demo==1 \\\nother requirement\n", - "demo==1\n", - f"demo==1\n --hash=sha256:{'a' * 64}\n", - f"demo==1 \\\n --hash=sha256:{'a' * 64} \\\n", - f"demo==1 \\\n --hash=sha256:{'a' * 64} junk\n", - f"demo==1 \\\n --hash=sha256:{'a' * 64} \\\n" f" --hash=sha256:{'a' * 64}\n", - ), -) -def test_lockfile_validation_rejects_unpinned_or_malformed_records(tmp_path, invalid_lock): - lock = tmp_path / "invalid-lock.txt" - lock.write_text(invalid_lock, encoding="utf-8") - - with pytest.raises(AssertionError): - _lock_versions(lock) - - -def test_runtime_and_build_requirements_flow_into_combined_locks(): - runtime = _requirement_names(_active_lines(ROOT / "requirements.txt")) - linux_build = _lock_versions(ENG / "requirements-build-linux.txt").keys() - linux_test = _lock_versions(ENG / "requirements-test-linux.txt").keys() - macos = _lock_versions(ENG / "requirements-build-macos.txt").keys() - - assert _active_lines(ENG / "requirements-test-linux.in") == [ - "-r requirements-build-linux.in", - "-c requirements-build-linux.txt", - "-r ../requirements.txt", - ] - assert linux_build <= linux_test - assert runtime <= linux_test - assert runtime <= macos - - -def test_macos_lock_preserves_python_310_cryptography_compatibility(): - assert "cryptography<49" in _active_lines(ENG / "requirements-build-macos.in") - version = _lock_versions(ENG / "requirements-build-macos.txt")["cryptography"] - assert int(version.split(".", maxsplit=1)[0]) < 49 - - -@pytest.mark.parametrize("pipeline_path, locks", PIPELINE_LOCKS.items()) -def test_release_pipelines_only_use_locked_requirements(pipeline_path, locks): - pipeline = (ROOT / pipeline_path).read_text(encoding="utf-8") - installs = [ - line.strip() - for line in pipeline.splitlines() - if "pip install" in line and not line.lstrip().startswith("#") - ] - - for lock, expected_count in locks.items(): - matching = [line for line in installs if f"-r {lock}" in line] - assert len(matching) == expected_count - assert all("--require-hashes" in line for line in matching) - - for command in installs: - if " -r " not in command: - assert '"$WHEEL"' in command or "--no-index --find-links" in command - - -def test_linux_runtime_lock_is_installed_before_each_product_wheel(): - pipeline = (ROOT / "OneBranchPipelines/stages/build-linux-single-stage.yml").read_text( - encoding="utf-8" - ) - runtime_install = ( - "$PY -m pip install -q --require-hashes " "-r /workspace/eng/requirements-test-linux.txt;" - ) - wheel_install = '$PY -m pip install -q "$WHEEL";' - runtime_positions = [ - match.start() for match in re.finditer(re.escape(runtime_install), pipeline) - ] - wheel_positions = [match.start() for match in re.finditer(re.escape(wheel_install), pipeline)] - - assert len(runtime_positions) == len(wheel_positions) == 2 - assert runtime_positions[0] < wheel_positions[0] < runtime_positions[1] < wheel_positions[1] - - -def test_refresh_workflow_is_pr_safe_and_immutable(workflow): - before_pr_job, pr_job = workflow.split(" open-pull-request:", maxsplit=1) - trigger_paths = ( - ".github/workflows/refresh-build-dependencies.yml", - "eng/requirements-build-*.in", - "eng/requirements-build-*.txt", - "eng/requirements-test-linux.in", - "eng/requirements-test-linux.txt", - "requirements.txt", - "OneBranchPipelines/stages/build-*-single-stage.yml", - "OneBranchPipelines/stages/build-odbc-all-stage.yml", - ) - - assert "pull_request_target:" not in workflow - assert all(f" - {path}" in workflow for path in trigger_paths) - assert 'cron: "0 8 * * 1"' in workflow - assert "timezone: America/Los_Angeles" in workflow - assert "group: refresh-release-build-dependencies" in workflow - assert "cancel-in-progress: false" in workflow - assert "permissions:\n contents: read" in workflow - assert "contents: write" not in before_pr_job - assert "if: github.event_name != 'pull_request'" in pr_job - assert "contents: write" in pr_job - assert "pull-requests: write" in pr_job - assert workflow.count("persist-credentials: false") == 2 - assert workflow.count("github.event.pull_request.head.sha || 'main'") == 2 - - actions = ACTION.findall(workflow) - assert actions - assert all(re.fullmatch(r"[0-9a-f]{40}", revision) for _, revision in actions) - - -def test_refresh_workflow_compiles_and_verifies_committed_locks(workflow): - compile_jobs = _section(workflow, " compile-linux:", "\n validate-locks:") - compile_commands = [ - line.strip() for line in compile_jobs.splitlines() if "uv pip compile" in line - ] - conditional_upgrade = "${{ github.event_name != 'pull_request' && '--upgrade' || '' }}" - required_options = ( - conditional_upgrade, - "--generate-hashes", - "--no-emit-index-url", - "--no-header", - "--strip-extras", - "--python-version 3.10", - "--default-index https://pypi.org/simple", - ) - - assert len(compile_commands) == 3 - assert all( - all(option in command for option in required_options) for command in compile_commands - ) - assert "uv pip compile --upgrade" not in workflow - assert "eng/requirements-build-linux.txt eng/requirements-build-linux.in" in compile_commands[0] - assert "eng/requirements-test-linux.txt eng/requirements-test-linux.in" in compile_commands[1] - assert '"${{ matrix.output }}" "${{ matrix.input }}"' in compile_commands[2] - - for name, runner, input_path, output_path in ( - ( - "macos", - "macos-latest", - "eng/requirements-build-macos.in", - "eng/requirements-build-macos.txt", - ), - ( - "windows", - "windows-latest", - "eng/requirements-build-windows.in", - "eng/requirements-build-windows.txt", - ), - ( - "odbc", - "windows-latest", - "eng/requirements-build-odbc.in", - "eng/requirements-build-odbc.txt", - ), - ): - item = _matrix_item(compile_jobs, name) - assert f"os: {runner}" in item - assert f"input: {input_path}" in item - assert f"output: {output_path}" in item - - assert compile_jobs.count("if: github.event_name == 'pull_request'") == 2 - assert "git ls-files --error-unmatch -- eng/requirements-build-linux.txt" in compile_jobs - assert 'git ls-files --error-unmatch -- "${{ matrix.output }}"' in compile_jobs - assert "git diff --exit-code -- eng/requirements-build-linux.txt" in compile_jobs - assert 'git diff --exit-code -- "${{ matrix.output }}"' in compile_jobs - - -@pytest.mark.parametrize("name, expected", VALIDATION_MATRIX.items()) -def test_refresh_workflow_validates_each_release_target(workflow, name, expected): - validation = _section(workflow, " validate-locks:", "\n open-pull-request:") - artifact, lock, versions, platforms = expected - item = _matrix_item(validation, name) - - assert f"artifact: {artifact}" in item - assert f"lock: {lock}" in item - assert f'versions: "{versions}"' in item - assert f'platforms: "{platforms}"' in item - - for option in ( - "--dry-run", - "--no-cache", - "--only-binary :all:", - "--require-hashes", - '--python-version "$version"', - '--python-platform "$platform"', - ): - assert option in validation - - -def test_refresh_workflow_opens_one_tracked_update_pr(workflow): - pr_job = _section(workflow, " open-pull-request:") - - assert "needs: validate-locks" in pr_job - assert "ref: main" in pr_job - assert "BUILD_DEPENDENCY_WORK_ITEM:" in pr_job - assert "> AB#${BUILD_DEPENDENCY_WORK_ITEM}" in pr_job - assert 'branch="automation/refresh-build-dependencies"' in pr_job - assert "git diff --quiet -- eng/requirements-build-*.txt" in pr_job - assert '--force-with-lease="refs/heads/$branch:$remote_sha"' in pr_job - pr_lookup = ( - 'pr_count="$(gh pr list --head "$branch" --base main ' - '--state open --json number --jq length)"' - ) - assert pr_lookup in pr_job - assert 'if [ "$pr_count" = "0" ]; then' in pr_job - assert 'if [ "$(gh pr list' not in pr_job - assert "gh pr create \\" in pr_job