Bundle OSS C++ bridge in Linux Python wheels

.github/workflows/ci.yml

@@ -360,17 +360,9 @@ jobs:
         if: matrix.suite == 'oss-cpp'
         run: |
           sudo apt-get update
-          sudo apt-get install -y libcurl4-openssl-dev libssl-dev
-          cmake -S .oss-cpp-sdk -B build/oss-sdk -DCMAKE_BUILD_TYPE=Release \
-            -DUSE_SYSTEM_CURL=ON -DUSE_SYSTEM_OPENSSL=ON -DBUILD_SHARED_LIBS=ON \
-            -DCMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized \
-            -DCMAKE_INSTALL_PREFIX="$PWD/build/oss-install"
-          cmake --build build/oss-sdk -j 2
-          cmake --install build/oss-sdk
-          cmake -S integrations/oss-cpp -B build/oss-cpp \
-            -DCMAKE_PREFIX_PATH="$PWD/build/oss-install"
-          cmake --build build/oss-cpp -j 2
-          echo "OSS_CPP_BRIDGE_LIBRARY=$PWD/build/oss-cpp/liboss_cpp_bridge.so" >> "$GITHUB_ENV"
+          sudo apt-get install -y libssl-dev
+          bash scripts/build_python_oss_cpp_bridge.sh x86_64-unknown-linux-gnu
+          echo "OSS_CPP_BRIDGE_LIBRARY=$PWD/bindings/python/python/pypaimon_rust/liboss_cpp_bridge.so" >> "$GITHUB_ENV"
 
       - name: OSS C++ SDK Integration Test
         if: matrix.suite == 'oss-cpp'

.github/workflows/release_python_binding.yml

@@ -136,6 +136,14 @@ jobs:
           cargo fetch --locked
           python3 scripts/release_licenses.py --stage-python ${{ matrix.rust_target }}
 
+      - name: Checkout OSS C++ SDK
+        if: runner.os == 'Linux'
+        uses: actions/checkout@v7
+        with:
+          repository: aliyun/alibabacloud-oss-cpp-sdk-v2
+          ref: c13a82038b0696f99c32f4e86b623ffcc9e62140
+          path: .oss-cpp-sdk
+
       - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
         with:
           maturin-version: v1.14.1
@@ -147,7 +155,7 @@ jobs:
           before-script-linux: |
             # Install OpenSSL for x86_64 (manylinux2014 with OpenSSL 1.1)
             if [ "${{ matrix.rust_target }}" = "x86_64-unknown-linux-gnu" ]; then
-              yum install -y openssl11-devel
+              yum install -y openssl11-devel openssl11-static
               # Create symlinks so pkg-config finds openssl11 as openssl
               ln -sf /usr/lib64/pkgconfig/openssl11.pc /usr/lib64/pkgconfig/openssl.pc
               ln -sf /usr/lib64/pkgconfig/libssl11.pc /usr/lib64/pkgconfig/libssl.pc
@@ -206,9 +214,29 @@ jobs:
               export CFLAGS_aarch64_unknown_linux_gnu="-D__ARM_ARCH=8"
             fi
 
+            bash ../../scripts/build_python_oss_cpp_bridge.sh \
+              "${{ matrix.rust_target }}"
+
       - name: Verify wheel
         run: python scripts/verify_python_wheels.py --allow-partial bindings/python/dist
 
+      - name: Load bundled OSS C++ bridge
+        if: matrix.rust_target == 'x86_64-unknown-linux-gnu'
+        run: |
+          python3 - <<'PY'
+          import ctypes
+          import glob
+          import tempfile
+          import zipfile
+          from pathlib import Path
+
+          wheel, = glob.glob('bindings/python/dist/*.whl')
+          with zipfile.ZipFile(wheel) as archive, tempfile.TemporaryDirectory() as temp:
+              bridge = Path(temp) / 'liboss_cpp_bridge.so'
+              bridge.write_bytes(archive.read('pypaimon_rust/liboss_cpp_bridge.so'))
+              assert ctypes.CDLL(str(bridge)).oss_cpp_bridge_abi_version() == 2
+          PY
+
       - name: Upload wheels
         uses: actions/upload-artifact@v7
         with:

.licenserc.yaml

@@ -31,6 +31,8 @@ header:
     - "bindings/go/tests/testdata/**"
     - "third-party-licenses/jieba-rs-0.10.3.LICENSE"
     - "third-party-licenses/openssl-1.1.1.LICENSE"
+    - "third-party-licenses/oss-cpp-sdk-v2.THIRD_PARTY_NOTICES"
+    - "third-party-licenses/curl-8.13.0.COPYING"
     - "**/go.sum"
     - "**/DEPENDENCIES.*.tsv"
     - ".devcontainer/devcontainer.json"

bindings/python/pyproject.toml

@@ -54,6 +54,7 @@ include = [
     { path = "NOTICE", format = "wheel" },
     { path = "THIRD-PARTY-LICENSES.html", format = "wheel" },
     { path = "python/pypaimon_rust/py.typed", format = ["sdist", "wheel"] },
+    { path = "python/pypaimon_rust/liboss_cpp_bridge.*", format = "wheel" },
 ]
 exclude = [
     { path = "licenses/**", format = "sdist" },

bindings/python/src/oss_cpp.rs

@@ -22,7 +22,9 @@ use std::path::{Path, PathBuf};
 const LIBRARY_PATH: &str = "fs.oss.cpp.library.path";
 
 pub(crate) fn complete_options(options: HashMap<String, String>) -> HashMap<String, String> {
-    complete_options_with(options, || discover_library("pypaimon_oss_cpp"))
+    complete_options_with(options, || {
+        discover_library("pypaimon_rust").or_else(|| discover_library("pypaimon_oss_cpp"))
+    })
 }
 
 fn complete_options_with(

integrations/oss-cpp/README.md

@@ -33,17 +33,16 @@ cmake --build build/oss-cpp
 cargo build -p paimon --features storage-oss-cpp
 ```
 
-The Python binding already includes this feature, but neither the bridge nor
-the SDK is bundled in its wheel. Install both on each worker. Tested on Linux
-with SDK 0.2.0 (`c13a82038b0696f99c32f4e86b623ffcc9e62140`).
+Linux `pypaimon-rust` wheels bundle the bridge and SDK. Source builds and other
+platforms still require a separate bridge. The bundled SDK is 0.2.0
+(`c13a82038b0696f99c32f4e86b623ffcc9e62140`).
 
 ## Configure
 
 Set the native table/catalog FileIO options:
 
 ```text
 fs.oss.impl = cpp
-fs.oss.cpp.library.path = /absolute/path/liboss_cpp_bridge.so
 fs.oss.endpoint = https://oss-cn-shanghai-internal.aliyuncs.com
 fs.oss.region = cn-shanghai
 ```
@@ -52,10 +51,9 @@ Use the existing `fs.oss.accessKeyId`, `fs.oss.accessKeySecret` and optional
 `fs.oss.securityToken` credentials. REST catalog credentials still refresh;
 static FileIO credentials do not.
 
-Python bindings also discover `liboss_cpp_bridge.so` (Linux) or
-`liboss_cpp_bridge.dylib` (macOS) beside `pypaimon_oss_cpp/__init__.py`.
-An explicit `fs.oss.cpp.library.path` takes precedence. The companion package
-is not yet published.
+Python bindings discover the bridge in `pypaimon_rust`, then in
+`pypaimon_oss_cpp` if installed. Set `fs.oss.cpp.library.path` only to override
+discovery or for a source build.
 
 | Option | Default |
 |---|---:|

scripts/build_python_oss_cpp_bridge.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+target="${1:?pass the Rust target}"
+root="$(cd "$(dirname "$0")/.." && pwd)"
+sdk="$root/.oss-cpp-sdk"
+build="$root/build/python-oss-cpp-$target"
+package="$root/bindings/python/python/pypaimon_rust"
+
+test -f "$sdk/CMakeLists.txt" || { echo "Missing pinned OSS C++ SDK checkout" >&2; exit 1; }
+
+compiler=()
+if [[ "$target" == aarch64-unknown-linux-gnu ]]; then
+  compiler=(-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc
+            -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++
+            -DOPENSSL_ROOT_DIR=/usr/aarch64-unknown-linux-gnu)
+  strip=aarch64-linux-gnu-strip
+elif [[ "$target" != x86_64-unknown-linux-gnu ]]; then
+  echo "Unsupported OSS C++ wheel target: $target" >&2
+  exit 1
+else
+  strip=strip
+fi
+
+cmake -S "$sdk" -B "$build/sdk" \
+  "${compiler[@]}" \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="$build/install" \
+  -DCMAKE_INSTALL_LIBDIR=lib \
+  -DOPENSSL_USE_STATIC_LIBS=ON \
+  -DUSE_SYSTEM_OPENSSL=ON \
+  -DUSE_SYSTEM_CURL=OFF \
+  -DBUILD_SHARED_LIBS=OFF \
+  -DCMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized
+cmake --build "$build/sdk" --parallel 2
+cmake --install "$build/sdk"
+
+cmake -S "$root/integrations/oss-cpp" -B "$build/bridge" \
+  "${compiler[@]}" \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_PREFIX_PATH="$build/install" \
+  -DOPENSSL_USE_STATIC_LIBS=ON \
+  -DCMAKE_SHARED_LINKER_FLAGS=-Wl,--no-undefined \
+  -Wno-dev
+cmake --build "$build/bridge" --parallel 2
+
+bridge="$build/bridge/liboss_cpp_bridge.so"
+if readelf -d "$bridge" | grep -Eq 'Shared library: \[lib(ssl|crypto|curl)'; then
+  echo "OSS C++ bridge still depends on system OpenSSL or curl" >&2
+  exit 1
+fi
+cp "$bridge" "$package/liboss_cpp_bridge.so"
+"$strip" --strip-unneeded "$package/liboss_cpp_bridge.so"

scripts/release_licenses.py

@@ -589,6 +589,32 @@ def bundled_component_html(
     )
 
 
+def oss_cpp_wheel_licenses(root: Path, report: Report) -> str:
+    if report.component != "python" or not report.target.endswith("linux-gnu"):
+        return ""
+    sdk_notices = (root / "third-party-licenses/oss-cpp-sdk-v2.THIRD_PARTY_NOTICES").read_text(
+        encoding="utf-8"
+    )
+    curl_license = (root / "third-party-licenses/curl-8.13.0.COPYING").read_text(
+        encoding="utf-8"
+    )
+    openssl_version = "1.1.1k FIPS" if report.target.startswith("x86_64") else "1.1.1w"
+    return "\n".join(
+        [
+            "",
+            "        <h2>OSS C++ bridge dependencies</h2>",
+            '        <h3 id="bundled-oss-cpp-sdk">Alibaba Cloud OSS C++ SDK v2</h3>',
+            "        <p>Version 0.2.0, commit c13a82038b0696f99c32f4e86b623ffcc9e62140;",
+            "        Apache-2.0 (full text in LICENSE). Bundled SDK notices:</p>",
+            f'        <pre class="license-text">{html.escape(sdk_notices)}</pre>',
+            '        <h3 id="bundled-curl-8.13.0">curl 8.13.0</h3>',
+            f'        <pre class="license-text">{html.escape(curl_license)}</pre>',
+            f"        <p>OpenSSL {openssl_version} is also linked statically into the bridge;",
+            "        its license is listed above.</p>",
+        ]
+    )
+
+
 def complete_report(
     root: Path, base_report: str, report: Report, metadata: dict
 ) -> str:
@@ -628,6 +654,7 @@ def complete_report(
     result = (
         result[:closing_main]
         + bundled_component_html(root, report, result, metadata)
+        + oss_cpp_wheel_licenses(root, report)
         + "\n"
         + result[closing_main:]
     )

scripts/tests/test_release_licenses.py

@@ -22,8 +22,10 @@
 from scripts.release_licenses import (
     LicenseCorrection,
     PYTHON_MIT_CORRECTIONS,
+    Report,
     binary_notice,
     correction_html,
+    oss_cpp_wheel_licenses,
     resolved_packages,
 )
 
@@ -217,5 +219,19 @@ def test_notice_uses_only_normal_dependencies(self) -> None:
             self.assertNotIn("Optional dependency notice", notice)
 
 
+class OssCppWheelLicensesTest(unittest.TestCase):
+    def test_only_linux_python_wheels_include_sdk_and_curl(self) -> None:
+        root = Path(__file__).resolve().parents[2]
+        for target, expected in (
+            ("x86_64-unknown-linux-gnu", True),
+            ("aarch64-unknown-linux-gnu", True),
+            ("aarch64-apple-darwin", False),
+        ):
+            report = Report("python", "bindings/python/Cargo.toml", target, "unused")
+            text = oss_cpp_wheel_licenses(root, report)
+            self.assertEqual('id="bundled-oss-cpp-sdk"' in text, expected)
+            self.assertEqual('id="bundled-curl-8.13.0"' in text, expected)
+
+
 if __name__ == "__main__":
     unittest.main()

scripts/verify_python_wheels.py

@@ -199,6 +199,9 @@ def verify_license_report(content: bytes, target: str, description: str) -> None
     )
     for marker in markers:
         require(marker in content, f"{description} is missing {marker!r}")
+    if target.endswith("linux-gnu"):
+        for marker in (b'id="bundled-oss-cpp-sdk"', b'id="bundled-curl-8.13.0"'):
+            require(marker in content, f"{description} is missing {marker!r}")
 
 
 def parse_metadata(content: bytes, description: str) -> Message:
@@ -466,6 +469,16 @@ def verify_wheel(path: Path) -> tuple[str, str]:
             name for name in names if name != native_member and is_native_library(name)
         ]
         if target.endswith("linux-gnu"):
+            bridge = "pypaimon_rust/liboss_cpp_bridge.so"
+            require(
+                bridge in extra_native_members,
+                f"wheel is missing OSS C++ bridge: {path.name}",
+            )
+            with archive.open(bridge) as bridge_file:
+                verify_native_header(
+                    bridge_file.read(65536), target, f"{path.name}:{bridge}"
+                )
+            extra_native_members.remove(bridge)
             openssl_version = (
                 b"OpenSSL 1.1.1k  FIPS"
                 if target.startswith("x86_64")

third-party-licenses/README.md

@@ -30,6 +30,10 @@ The XZ Utils 5.8.3 license is read from the locked `liblzma-sys` crate at
 `c62e0df1f9dcc2cc1e014711c5aa4561ae260538`. It covers `jieba-rs` 0.10.3 and
 `jieba-macros` 0.10.3.
 
+Linux Python wheels also bundle OSS C++ SDK v2 at
+`c13a82038b0696f99c32f4e86b623ffcc9e62140` and curl 8.13.0.
+Their notices and curl license are copied here from those pinned sources.
+
 Target-specific Python and Go reports are generated in their binary build jobs.
 They are intentionally not committed or included in the ASF source archive.
 
@@ -38,3 +42,5 @@ Sources:
 - <https://github.com/openssl/openssl/blob/OpenSSL_1_1_1k/LICENSE>
 - <https://github.com/openssl/openssl/blob/OpenSSL_1_1_1w/LICENSE>
 - <https://github.com/messense/jieba-rs/blob/c62e0df1f9dcc2cc1e014711c5aa4561ae260538/LICENSE>
+- <https://github.com/aliyun/alibabacloud-oss-cpp-sdk-v2/blob/c13a82038b0696f99c32f4e86b623ffcc9e62140/THIRD_PARTY_NOTICES>
+- <https://github.com/curl/curl/blob/curl-8_13_0/COPYING>

third-party-licenses/curl-8.13.0.COPYING

@@ -0,0 +1,22 @@
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright (c) 1996 - 2025, Daniel Stenberg, <daniel@haxx.se>, and many
+contributors, see the THANKS file.
+
+All rights reserved.
+
+Permission to use, copy, modify, and distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright
+notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN
+NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+OR OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall not
+be used in advertising or otherwise to promote the sale, use or other dealings
+in this Software without prior written authorization of the copyright holder.