FIX: Enforce bundled native compatibility in Conda packages

OneBranchPipelines/scripts/build_conda_packages.py

@@ -4,7 +4,7 @@
 Replaces build-conda-packages.ps1 + build-conda-packages.sh (the same 7-step pipeline
 written twice, which had already drifted). conda is Python and every agent has a bootstrap
 interpreter, so ONE orchestrator runs on every leg; the platform differences (the Miniforge
-installer, the win-arm64 Terms-of-Service auto-accept, the Linux-only reachability gate) are
+installer, the win-arm64 channel profile, the Linux-only reachability gate) are
 a handful of branches, not a second 360-line script. Running as a NORMAL process also means
 the caller reads the exit code directly -- so the PowerShell ErrorActionPreference flips, the
 `2>$null` swallows, and the `cmd /c "exit 0"` reset all disappear.
@@ -297,6 +297,7 @@ def build_env(
 ) -> dict[str, str]:
     """The environment consumed by the recipe (jinja + build.sh/bld.bat) and by conda-build."""
     env = dict(os.environ)
+    env.pop("CONDA_PLUGINS_AUTO_ACCEPT_TOS", None)
     env["WHEELS_DIR"] = links
     env["MSSQL_PYTHON_VERSION"] = mssql_ver
     env["MSSQL_ODBC_VERSION"] = odbc_ver
@@ -306,12 +307,6 @@ def build_env(
         # host can execute it (natively / Rosetta 2 / QEMU binfmt).
         env["CONDA_SUBDIR"] = cross_target_subdir
         _log(f"Cross-targeting conda subdir: CONDA_SUBDIR={cross_target_subdir}")
-        if cross_target_subdir == "win-arm64":
-            # win-arm64 deps (python 3.12-3.14, cryptography, vc14_runtime, pyodbc) live on
-            # Anaconda `defaults`, not conda-forge. Auto-accept the defaults ToS so the
-            # unattended host-env + verify solves never block on a prompt.
-            env["CONDA_PLUGINS_AUTO_ACCEPT_TOS"] = "yes"
-            _log("win-arm64: CONDA_PLUGINS_AUTO_ACCEPT_TOS=yes")
         if cross_target_subdir.endswith("aarch64") and os.path.isdir("/usr/aarch64-linux-gnu"):
             # Emulated aarch64 verify runs under qemu-user; point it at the aarch64 glibc loader.
             env.setdefault("QEMU_LD_PREFIX", "/usr/aarch64-linux-gnu")
@@ -392,17 +387,16 @@ def audit_packages(
         env=env,
         what="RUNPATH self-containment audit",
     )
-    # win-arm64 is cross-built on x64 where its runtime import is skipped, so its arch is
-    # trusted from the wheel filename UNLESS the PE machine assert reads it out of the payload.
-    if target_subdir == "win-arm64":
+    # Both Windows packages must retain the core; cross builds also rely on static architecture.
+    if target_subdir in ("win-64", "win-arm64"):
         pe = os.path.join(eng, "assert_pe_machine.py")
         if not os.path.isfile(pe):
             _die(f"PE machine-type assert script not found at {pe}")
-        _log("=== win-arm64 PE machine-type assert (vendored .pyd/.dll must be ARM64) ===")
+        _log(f"=== {target_subdir} PE machine-type and required native-component assert ===")
         run(
-            [conda, "run", "-n", builder, "python", pe, "--root", bld, "--subdir", "win-arm64"],
+            [conda, "run", "-n", builder, "python", pe, "--root", bld, "--subdir", target_subdir],
             env=env,
-            what="win-arm64 PE machine-type assert",
+            what=f"{target_subdir} PE machine-type assert",
         )
     # osx legs: verify the universal binding contains the target slice and each thin vendored
     # driver dylib matches its architecture-specific directory. osx-arm64 is cross-built on the
@@ -470,14 +464,27 @@ def _import_probe(mod_name: str, ok_label: str) -> str:
     INSTALLED package, not the checkout. Uses abspath (NOT realpath) so conda's softlink install
     mode -- where the site-packages entry symlinks into the pkgs/ cache OUTSIDE the prefix -- is
     not false-failed: the import PATH stays under the prefix regardless of hard/soft link; only
-    the symlink TARGET would not. Then prints ok_label + the version."""
+    the symlink TARGET would not. Then prints ok_label + the installed module path."""
     return (
         f"import os,sys,{mod_name} as m;"
         "f=os.path.normcase(os.path.abspath(m.__file__));"
         "pref=os.path.normcase(os.path.abspath(sys.prefix));"
         f"assert f.startswith(pref+os.sep),{mod_name!r}+' loaded from '+m.__file__+"
         "', not under the conda env '+sys.prefix+' (stray PYTHONPATH/.pth?)';"
-        f"print({ok_label!r},m.__version__)"
+        f"print({ok_label!r},m.__file__)"
+    )
+
+
+def _core_probe() -> str:
+    return (
+        _import_probe("mssql_py_core", "CORE_PACKAGE_OK") + ";import importlib.machinery;"
+        "exts=[v for k,v in list(sys.modules.items()) "
+        "if (k=='mssql_py_core' or k.startswith('mssql_py_core.')) "
+        "and isinstance(getattr(v,'__loader__',None),importlib.machinery.ExtensionFileLoader)];"
+        "assert exts,'mssql_py_core did not load its required native extension';"
+        "assert all(os.path.normcase(os.path.abspath(v.__file__)).startswith(pref+os.sep) "
+        "for v in exts),'core native extension loaded outside installed prefix';"
+        "print('CORE_NATIVE_OK',*[v.__file__ for v in exts])"
     )
 
 
@@ -610,6 +617,12 @@ def _verify_impl(
                 f"arm64 cross-build. Refusing to silently skip validation. Output: {out}"
             )
 
+        # A separate process prevents API/driver preloads from masking core load failures.
+        run(
+            [conda, "run", "-n", name, "python", "-c", _core_probe()],
+            env=env,
+            what=f"independent required mssql_py_core load (py {py})",
+        )
         _log(f"=== [py {py}] import mssql_python + prove the vendored ODBC payload is present ===")
         run(
             [

README.md

@@ -12,20 +12,26 @@ The driver is compatible with all the Python versions >= 3.10
 > **Important Note:**
 >
 > ### ODBC Driver Distribution
-> The ODBC driver binaries used by `mssql-python` are distributed exclusively through a dedicated companion package:
+> For **pip/PyPI installations**, the ODBC driver binaries used by `mssql-python` are distributed through a dedicated companion distribution:
 >
 > - Package: `mssql-python-odbc`
 > - Import name: `mssql_python_odbc`
 > - Current version: **18.6.2.1**
 >
 > `mssql-python` depends on `mssql-python-odbc==18.6.2.1`. The ODBC driver is loaded lazily when the first connection is created. `pip install mssql-python` transparently pulls the companion package alongside it — no separate install step is required.
 >
-> Starting with v1.13.0, the bundled `libs/` fallback that shipped in v1.12.0 has been removed. Creating a connection will fail if `mssql-python-odbc` is not installed. If you install `mssql-python` from a private index or with `--no-deps`, make sure `mssql-python-odbc==18.6.2.1` is installed alongside it.
+> Starting with v1.13.0, the bundled `libs/` fallback that shipped in v1.12.0 has been removed. For pip installations, creating a connection will fail if `mssql-python-odbc` is not installed. If you install `mssql-python` from a private index or with `--no-deps`, make sure `mssql-python-odbc==18.6.2.1` is installed alongside it.
+>
+> The **temporary Conda candidate** instead combines the code and ODBC payload in one `mssql-python` Conda package. It does not require a separately installed Conda ODBC package. This is not an announcement of public channel availability or release qualification; see the [Conda installation, migration, and readiness guide](conda/README.md).
 >
 > ### ODBC Provider Selection (opt-in)
 > `mssql-python` also supports selecting an alternate native ODBC provider before the first connection, via the `mssql_python.native_provider` module property or the `MSSQL_PYTHON_NATIVE_PROVIDER` environment variable (which takes precedence). A conflicting property assignment emits a `RuntimeWarning`. The default, `"msodbcsql18"`, is unchanged; opting into `"mssql-odbc"` requires the `mssql-python-rs` package (which bundles the Rust ODBC driver alongside the Rust TDS core). Call `mssql_python.get_native_provider_info()` to check the selected provider, source, package version, and resolved driver path.
 
 ## Installation
+
+The pip commands below describe the public release. The temporary combined Conda
+candidate has a separate platform matrix and Linux compatibility floor; it is not
+covered by the public release's production-readiness statement above.
 
 **Windows:** mssql-python can be installed with [pip](http://pypi.python.org/pypi/pip)
 ```bash
@@ -60,6 +66,30 @@ tdnf distro-sync && tdnf install -y libtool-ltdl krb5-libs glibc-iconv
 pip install mssql-python
 ```
 
+**Conda candidate:** Obtain the exact candidate channel and version from its owner;
+these changes do not publish packages to the public `microsoft` channel. The candidate
+includes the ODBC Driver 18 payload and required bulk-copy core. Linux requires
+**glibc >=2.34** for that complete payload; `krb5`, OpenSSL, and `libltdl` resolve from
+`conda-forge`, so the system package steps above are not required. Windows uses SChannel.
+On macOS, encrypted connections still require system OpenSSL from Homebrew
+(`brew install openssl`) or MacPorts, not Conda OpenSSL. Windows ARM64 dependencies
+resolve from `defaults`; handle channel terms separately, without automatic acceptance.
+Use a fresh environment and replace the placeholders with the owner-provided values:
+```bash
+# Windows x64, macOS, and Linux
+conda install -c "<candidate-channel>" -c microsoft -c conda-forge --strict-channel-priority --override-channels "mssql-python=<candidate-version>"
+
+# Windows ARM64
+conda install -c "<candidate-channel>" -c microsoft -c defaults --override-channels "mssql-python=<candidate-version>"
+```
+
+**Conda release status:** Publication tooling and its administrator prerequisites are
+proposed separately in the [release-additions PR](https://github.com/microsoft/mssql-python/pull/720).
+These native-packaging changes do not publish packages or require a particular merge order.
+Static audits and import checks do not certify SQL, certificate-verified TLS, authentication,
+bulk copy, or optional features across the full matrix. Production publication remains gated
+on separate release controls and qualification; validate-only success does not authorize it.
+
 ## Key Features
 ### Supported Platforms
 

conda/README.md

@@ -0,0 +1,54 @@
+# Combined Conda candidate
+
+This recipe combines the matching code and ODBC wheels into **one `mssql-python`
+Conda package**, including the required bulk-copy core. Pip instead installs
+`mssql-python-odbc` as a separate companion distribution. Neither requires a
+separately installed ODBC driver or driver manager.
+
+Direct recipe builds must set `MSSQL_PYTHON_VERSION` to the exact selected code-wheel
+version before rendering/building. The shared orchestrator derives and supplies it
+automatically; omitted input fails recipe rendering instead of choosing a release.
+Both native installation and cross extraction require the bulk-copy initializer and
+a compatible extension filename. The separate native audits still validate binary headers.
+
+This is a temporary candidate, not an announcement of public channel availability.
+Obtain the exact candidate archive/channel from its owner and install into a new
+Conda environment. Activate it and select the same interpreter/kernel in your IDE
+or notebook. Application imports and the public API remain unchanged. Use Conda
+for upgrades; do not overwrite Conda-owned driver files with pip.
+
+Linux requires **glibc >=2.34** for the complete native payload, including the
+bulk-copy core, even when the binding wheel has a lower platform tag. Do not force
+installation on older glibc. This does not change the separate PyPI support claim.
+macOS retains its external Homebrew/MacPorts OpenSSL prerequisite for encryption;
+Conda OpenSSL alone does not satisfy the driver's system-path lookup.
+
+The release goal is the matching PyPI release's public API and supported feature
+behavior on all 28 ordinary CPython variants: 3.10-3.14 on win-64, linux-64,
+linux-aarch64, osx-64 and osx-arm64; 3.12-3.14 on win-arm64. No silent removal of
+required native functionality is acceptable. Optional features require their
+corresponding dependencies; Windows ARM64 PyArrow availability remains a blocker
+to qualifying those features, not permission to drop them.
+
+These guards are **not full-matrix parity certification**. Native target execution,
+SQL, certificate-verified TLS, authentication, bulk-copy and optional-feature tests
+remain required before release. Cross-build/static checks and a DB-less driver
+load do not establish those results. Applicable OS, certificate and authentication
+configuration remain external prerequisites.
+
+Use only organizationally approved channels and handle applicable terms separately;
+the Windows ARM64 dependency profile includes Anaconda `defaults`. Before unattended
+Windows ARM64 builds, agent owners must use approved provisioning to put a disposable
+Conda installation on `PATH`, with applicable channel terms handled for the job's
+execution identity. The orchestrator's automatic Miniforge installation does not
+establish that approval. If Conda enforces terms that have not been handled, the build
+or verification solve stops with Conda's diagnostic; provision the prerequisite before
+running again. Automatic acceptance, including an inherited
+`CONDA_PLUGINS_AUTO_ACCEPT_TOS` opt-in, remains disabled.
+
+Run this existing build workflow only in a disposable isolated installation: shared-environment
+ownership hardening is outside this change. Publication/provenance tooling is proposed
+in a separate release-additions PR, without a required merge order; see the
+[release status and qualification caveats](../README.md#installation).
+Neither this native-packaging change nor validate-only success authorizes production
+publication.

conda/mssql-python/bld.bat

@@ -39,23 +39,22 @@ if errorlevel 1 (
     echo ERROR: extracted "!CODE_WHL!" has no mssql_python\ddbc_bindings.cp%CONDA_PY% pyd ^(wrong-Python binding^).
     exit /b 1
   )
-  REM Keep mssql_py_core when the wheel provides a matching-arch native ext so bulk copy
-  REM ships (PR #737 makes the win-arm64 wheel vendor the arm64 core). If only the legacy
-  REM x64 core is present (a pre-#737 wheel), strip it so the package never carries a core
-  REM that can't load on the target -- the .pyd name encodes the arch. Bulk copy then lazily
-  REM reports "not available"; the rest of the DBAPI works. Mirrors the ddbc check above.
-  if exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" (
-    echo Keeping matching-arch mssql_py_core; bulk copy enabled on the !ODBC_ARCH! package.
-  ) else (
-    echo No cp%CONDA_PY%-!ODBC_ARCH! mssql_py_core in the wheel; removing the mismatched core ^(bulk copy unavailable until the arm64-core wheel ships^).
-    if exist "%SP%\mssql_py_core" rmdir /s /q "%SP%\mssql_py_core"
-    if exist "%SP%\mssql_py_core.libs" rmdir /s /q "%SP%\mssql_py_core.libs"
-  )
 ) else (
   "%PYTHON%" -m pip install --no-deps --no-index --find-links "%WHEELS_DIR%" %PKG_NAME%==%PKG_VERSION% -vv
   if errorlevel 1 exit /b 1
 )
 
+REM Both install paths require bulk copy. The PE audit still checks actual architecture.
+if not exist "%SP%\mssql_py_core\__init__.py" (
+  echo ERROR: required mssql_py_core initializer is missing. Use a corrected upstream wheel; refusing reduced functionality.
+  exit /b 1
+)
+REM Bare .pyd is the Windows stable-ABI suffix.
+if not exist "%SP%\mssql_py_core\mssql_py_core.cp%CONDA_PY%-!ODBC_ARCH!.pyd" if not exist "%SP%\mssql_py_core\mssql_py_core.pyd" (
+  echo ERROR: required mssql_py_core is missing or incompatible with cp%CONDA_PY% !ODBC_ARCH!. Use a corrected upstream wheel; refusing reduced functionality.
+  exit /b 1
+)
+
 REM Extract the arch-specific odbc wheel into the SAME site-packages so
 REM mssql_python_odbc\libs\ sits beside mssql_python\ and the loader finds the driver.
 REM The py3-none tag only means "no Python bytecode" -- the vendored driver DLLs ARE

conda/mssql-python/build.sh

@@ -18,15 +18,19 @@ odbc_ver="${MSSQL_ODBC_VERSION:?MSSQL_ODBC_VERSION not set}"
 if "$PYTHON" -c "import sys" >/dev/null 2>&1; then
   "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "$PKG_NAME==$PKG_VERSION" -vv
   "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "mssql-python-odbc==$odbc_ver" -vv
+  core_suffix="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))')"
 else
+  core_suffix=".cpython-${CONDA_PY}-darwin.so"
   echo "Host Python '$PYTHON' is not executable on this agent (non-emulated cross-build);"
   echo "extracting both wheels into \$SP_DIR without running Python."
   mkdir -p "$SP_DIR"
   pkg_underscore="${PKG_NAME//-/_}"
   # universal2 wheels are cpXY-specific (compiled ddbc_bindings), so filter on the
   # target CONDA_PY to never grab another interpreter's wheel (mirrors bld.bat).
   code_whl=""
-  for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-cp${CONDA_PY}-"*.whl; do
+  # Match macosx explicitly (like the odbc glob below) so a stray Linux cpXY wheel staged in
+  # the same dir can never be picked up on this macOS-only cross branch.
+  for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-cp${CONDA_PY}-"*macosx*.whl; do
     [ -e "$w" ] && { code_whl="$w"; break; }
   done
   [ -n "$code_whl" ] || { echo "ERROR: no ${PKG_NAME}==${PKG_VERSION} cp${CONDA_PY} wheel in '$WHEELS_DIR'" >&2; exit 1; }
@@ -52,6 +56,15 @@ else
   unzip -oq "$odbc_whl" -d "$SP_DIR"
 fi
 
+# Both install paths require bulk copy; the platform audits still check binary headers.
+if [ ! -f "$SP_DIR/mssql_py_core/__init__.py" ] || {
+  [ ! -f "$SP_DIR/mssql_py_core/mssql_py_core${core_suffix}" ] &&
+  [ ! -f "$SP_DIR/mssql_py_core/mssql_py_core.abi3.so" ]
+}; then
+  echo "ERROR: required mssql_py_core initializer or compatible extension is missing. Use a corrected upstream wheel; refusing reduced functionality." >&2
+  exit 1
+fi
+
 # ---------------------------------------------------------------------------
 # Linux driver reachability (#563) -- the core fix.
 # ---------------------------------------------------------------------------

conda/mssql-python/meta.yaml

@@ -1,4 +1,4 @@
-{% set version = environ.get('MSSQL_PYTHON_VERSION', '1.14.0') %}
+{% set version = environ['MSSQL_PYTHON_VERSION'] %}
 
 package:
   name: mssql-python
@@ -45,6 +45,8 @@ requirements:
     - python
     # conda does NOT inherit the wheel's install_requires, so pin azure-identity here.
     - azure-identity >=1.12.0
+    # Defaults' Windows ARM64 Python 3.12 does not export its CPython ABI constraint.
+    - python_abi 3.12.* *_cp312  # [win and arm64 and py == 312]
     # ODBC Driver 18 payload deps (the driver ships inside this package, so its
     # security-serviced deps are declared here). OpenSSL for TLS is dlopen'd, so
     # overlinking can't see it; pinned <4 (Driver 18 supports the OpenSSL 1.1/3.0 ABI
@@ -63,8 +65,8 @@ requirements:
     # msodbcsql18.dll imports VCRUNTIME140.dll but the vendored vcredist ships only
     # msvcp140.dll; declare the serviced conda runtime.
     - vc14_runtime            # [win]
-    # Re-assert the wheel's platform floor (conda drops the wheel tag); never stricter.
-    - __glibc >=2.28      # [linux]
+    # The complete payload, including the required bulk-copy core, needs GLIBC_2.34.
+    - __glibc >=2.34      # [linux]
     - __osx >=15.0        # [osx]
 
 test: