diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d568322f7..fca7b5e96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -360,17 +360,9 @@ jobs: if: matrix.suite == 'oss-cpp' run: | sudo apt-get update - sudo apt-get install -y libcurl4-openssl-dev libssl-dev - cmake -S .oss-cpp-sdk -B build/oss-sdk -DCMAKE_BUILD_TYPE=Release \ - -DUSE_SYSTEM_CURL=ON -DUSE_SYSTEM_OPENSSL=ON -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized \ - -DCMAKE_INSTALL_PREFIX="$PWD/build/oss-install" - cmake --build build/oss-sdk -j 2 - cmake --install build/oss-sdk - cmake -S integrations/oss-cpp -B build/oss-cpp \ - -DCMAKE_PREFIX_PATH="$PWD/build/oss-install" - cmake --build build/oss-cpp -j 2 - echo "OSS_CPP_BRIDGE_LIBRARY=$PWD/build/oss-cpp/liboss_cpp_bridge.so" >> "$GITHUB_ENV" + sudo apt-get install -y libssl-dev + bash scripts/build_python_oss_cpp_bridge.sh x86_64-unknown-linux-gnu + echo "OSS_CPP_BRIDGE_LIBRARY=$PWD/bindings/python/python/pypaimon_rust/liboss_cpp_bridge.so" >> "$GITHUB_ENV" - name: OSS C++ SDK Integration Test if: matrix.suite == 'oss-cpp' diff --git a/.github/workflows/release_python_binding.yml b/.github/workflows/release_python_binding.yml index 79fc6d157..33a84a5a0 100644 --- a/.github/workflows/release_python_binding.yml +++ b/.github/workflows/release_python_binding.yml @@ -136,6 +136,14 @@ jobs: cargo fetch --locked python3 scripts/release_licenses.py --stage-python ${{ matrix.rust_target }} + - name: Checkout OSS C++ SDK + if: runner.os == 'Linux' + uses: actions/checkout@v7 + with: + repository: aliyun/alibabacloud-oss-cpp-sdk-v2 + ref: c13a82038b0696f99c32f4e86b623ffcc9e62140 + path: .oss-cpp-sdk + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: maturin-version: v1.14.1 @@ -147,7 +155,7 @@ jobs: before-script-linux: | # Install OpenSSL for x86_64 (manylinux2014 with OpenSSL 1.1) if [ "${{ matrix.rust_target }}" = "x86_64-unknown-linux-gnu" ]; then - yum install -y openssl11-devel + yum install -y openssl11-devel openssl11-static # Create symlinks so pkg-config finds openssl11 as openssl ln -sf /usr/lib64/pkgconfig/openssl11.pc /usr/lib64/pkgconfig/openssl.pc ln -sf /usr/lib64/pkgconfig/libssl11.pc /usr/lib64/pkgconfig/libssl.pc @@ -206,9 +214,29 @@ jobs: export CFLAGS_aarch64_unknown_linux_gnu="-D__ARM_ARCH=8" fi + bash ../../scripts/build_python_oss_cpp_bridge.sh \ + "${{ matrix.rust_target }}" + - name: Verify wheel run: python scripts/verify_python_wheels.py --allow-partial bindings/python/dist + - name: Load bundled OSS C++ bridge + if: matrix.rust_target == 'x86_64-unknown-linux-gnu' + run: | + python3 - <<'PY' + import ctypes + import glob + import tempfile + import zipfile + from pathlib import Path + + wheel, = glob.glob('bindings/python/dist/*.whl') + with zipfile.ZipFile(wheel) as archive, tempfile.TemporaryDirectory() as temp: + bridge = Path(temp) / 'liboss_cpp_bridge.so' + bridge.write_bytes(archive.read('pypaimon_rust/liboss_cpp_bridge.so')) + assert ctypes.CDLL(str(bridge)).oss_cpp_bridge_abi_version() == 2 + PY + - name: Upload wheels uses: actions/upload-artifact@v7 with: diff --git a/.licenserc.yaml b/.licenserc.yaml index 9ee03354c..2da21c290 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -31,6 +31,8 @@ header: - "bindings/go/tests/testdata/**" - "third-party-licenses/jieba-rs-0.10.3.LICENSE" - "third-party-licenses/openssl-1.1.1.LICENSE" + - "third-party-licenses/oss-cpp-sdk-v2.THIRD_PARTY_NOTICES" + - "third-party-licenses/curl-8.13.0.COPYING" - "**/go.sum" - "**/DEPENDENCIES.*.tsv" - ".devcontainer/devcontainer.json" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 131404e9b..2323e2fff 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -54,6 +54,7 @@ include = [ { path = "NOTICE", format = "wheel" }, { path = "THIRD-PARTY-LICENSES.html", format = "wheel" }, { path = "python/pypaimon_rust/py.typed", format = ["sdist", "wheel"] }, + { path = "python/pypaimon_rust/liboss_cpp_bridge.*", format = "wheel" }, ] exclude = [ { path = "licenses/**", format = "sdist" }, diff --git a/bindings/python/src/oss_cpp.rs b/bindings/python/src/oss_cpp.rs index ad2b4314f..7d0282b58 100644 --- a/bindings/python/src/oss_cpp.rs +++ b/bindings/python/src/oss_cpp.rs @@ -22,7 +22,9 @@ use std::path::{Path, PathBuf}; const LIBRARY_PATH: &str = "fs.oss.cpp.library.path"; pub(crate) fn complete_options(options: HashMap) -> HashMap { - complete_options_with(options, || discover_library("pypaimon_oss_cpp")) + complete_options_with(options, || { + discover_library("pypaimon_rust").or_else(|| discover_library("pypaimon_oss_cpp")) + }) } fn complete_options_with( diff --git a/integrations/oss-cpp/README.md b/integrations/oss-cpp/README.md index 0349a4728..b39d03d29 100644 --- a/integrations/oss-cpp/README.md +++ b/integrations/oss-cpp/README.md @@ -33,9 +33,9 @@ cmake --build build/oss-cpp cargo build -p paimon --features storage-oss-cpp ``` -The Python binding already includes this feature, but neither the bridge nor -the SDK is bundled in its wheel. Install both on each worker. Tested on Linux -with SDK 0.2.0 (`c13a82038b0696f99c32f4e86b623ffcc9e62140`). +Linux `pypaimon-rust` wheels bundle the bridge and SDK. Source builds and other +platforms still require a separate bridge. The bundled SDK is 0.2.0 +(`c13a82038b0696f99c32f4e86b623ffcc9e62140`). ## Configure @@ -43,7 +43,6 @@ Set the native table/catalog FileIO options: ```text fs.oss.impl = cpp -fs.oss.cpp.library.path = /absolute/path/liboss_cpp_bridge.so fs.oss.endpoint = https://oss-cn-shanghai-internal.aliyuncs.com fs.oss.region = cn-shanghai ``` @@ -52,10 +51,9 @@ Use the existing `fs.oss.accessKeyId`, `fs.oss.accessKeySecret` and optional `fs.oss.securityToken` credentials. REST catalog credentials still refresh; static FileIO credentials do not. -Python bindings also discover `liboss_cpp_bridge.so` (Linux) or -`liboss_cpp_bridge.dylib` (macOS) beside `pypaimon_oss_cpp/__init__.py`. -An explicit `fs.oss.cpp.library.path` takes precedence. The companion package -is not yet published. +Python bindings discover the bridge in `pypaimon_rust`, then in +`pypaimon_oss_cpp` if installed. Set `fs.oss.cpp.library.path` only to override +discovery or for a source build. | Option | Default | |---|---:| diff --git a/scripts/build_python_oss_cpp_bridge.sh b/scripts/build_python_oss_cpp_bridge.sh new file mode 100644 index 000000000..46ab33aec --- /dev/null +++ b/scripts/build_python_oss_cpp_bridge.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +target="${1:?pass the Rust target}" +root="$(cd "$(dirname "$0")/.." && pwd)" +sdk="$root/.oss-cpp-sdk" +build="$root/build/python-oss-cpp-$target" +package="$root/bindings/python/python/pypaimon_rust" + +test -f "$sdk/CMakeLists.txt" || { echo "Missing pinned OSS C++ SDK checkout" >&2; exit 1; } + +compiler=() +if [[ "$target" == aarch64-unknown-linux-gnu ]]; then + compiler=(-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc + -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ + -DOPENSSL_ROOT_DIR=/usr/aarch64-unknown-linux-gnu) + strip=aarch64-linux-gnu-strip +elif [[ "$target" != x86_64-unknown-linux-gnu ]]; then + echo "Unsupported OSS C++ wheel target: $target" >&2 + exit 1 +else + strip=strip +fi + +cmake -S "$sdk" -B "$build/sdk" \ + "${compiler[@]}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$build/install" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DOPENSSL_USE_STATIC_LIBS=ON \ + -DUSE_SYSTEM_OPENSSL=ON \ + -DUSE_SYSTEM_CURL=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized +cmake --build "$build/sdk" --parallel 2 +cmake --install "$build/sdk" + +cmake -S "$root/integrations/oss-cpp" -B "$build/bridge" \ + "${compiler[@]}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$build/install" \ + -DOPENSSL_USE_STATIC_LIBS=ON \ + -DCMAKE_SHARED_LINKER_FLAGS=-Wl,--no-undefined \ + -Wno-dev +cmake --build "$build/bridge" --parallel 2 + +bridge="$build/bridge/liboss_cpp_bridge.so" +if readelf -d "$bridge" | grep -Eq 'Shared library: \[lib(ssl|crypto|curl)'; then + echo "OSS C++ bridge still depends on system OpenSSL or curl" >&2 + exit 1 +fi +cp "$bridge" "$package/liboss_cpp_bridge.so" +"$strip" --strip-unneeded "$package/liboss_cpp_bridge.so" diff --git a/scripts/release_licenses.py b/scripts/release_licenses.py index 5785cf803..d87ed2ce9 100644 --- a/scripts/release_licenses.py +++ b/scripts/release_licenses.py @@ -589,6 +589,32 @@ def bundled_component_html( ) +def oss_cpp_wheel_licenses(root: Path, report: Report) -> str: + if report.component != "python" or not report.target.endswith("linux-gnu"): + return "" + sdk_notices = (root / "third-party-licenses/oss-cpp-sdk-v2.THIRD_PARTY_NOTICES").read_text( + encoding="utf-8" + ) + curl_license = (root / "third-party-licenses/curl-8.13.0.COPYING").read_text( + encoding="utf-8" + ) + openssl_version = "1.1.1k FIPS" if report.target.startswith("x86_64") else "1.1.1w" + return "\n".join( + [ + "", + "

OSS C++ bridge dependencies

", + '

Alibaba Cloud OSS C++ SDK v2

', + "

Version 0.2.0, commit c13a82038b0696f99c32f4e86b623ffcc9e62140;", + " Apache-2.0 (full text in LICENSE). Bundled SDK notices:

", + f' 
{html.escape(sdk_notices)}
', + '

curl 8.13.0

', + f' 
{html.escape(curl_license)}
', + f"

OpenSSL {openssl_version} is also linked statically into the bridge;", + " its license is listed above.

", + ] + ) + + def complete_report( root: Path, base_report: str, report: Report, metadata: dict ) -> str: @@ -628,6 +654,7 @@ def complete_report( result = ( result[:closing_main] + bundled_component_html(root, report, result, metadata) + + oss_cpp_wheel_licenses(root, report) + "\n" + result[closing_main:] ) diff --git a/scripts/tests/test_release_licenses.py b/scripts/tests/test_release_licenses.py index 2cfe1b8ae..162c2c446 100644 --- a/scripts/tests/test_release_licenses.py +++ b/scripts/tests/test_release_licenses.py @@ -22,8 +22,10 @@ from scripts.release_licenses import ( LicenseCorrection, PYTHON_MIT_CORRECTIONS, + Report, binary_notice, correction_html, + oss_cpp_wheel_licenses, resolved_packages, ) @@ -217,5 +219,19 @@ def test_notice_uses_only_normal_dependencies(self) -> None: self.assertNotIn("Optional dependency notice", notice) +class OssCppWheelLicensesTest(unittest.TestCase): + def test_only_linux_python_wheels_include_sdk_and_curl(self) -> None: + root = Path(__file__).resolve().parents[2] + for target, expected in ( + ("x86_64-unknown-linux-gnu", True), + ("aarch64-unknown-linux-gnu", True), + ("aarch64-apple-darwin", False), + ): + report = Report("python", "bindings/python/Cargo.toml", target, "unused") + text = oss_cpp_wheel_licenses(root, report) + self.assertEqual('id="bundled-oss-cpp-sdk"' in text, expected) + self.assertEqual('id="bundled-curl-8.13.0"' in text, expected) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/verify_python_wheels.py b/scripts/verify_python_wheels.py index a01242f33..a45c1b495 100644 --- a/scripts/verify_python_wheels.py +++ b/scripts/verify_python_wheels.py @@ -199,6 +199,9 @@ def verify_license_report(content: bytes, target: str, description: str) -> None ) for marker in markers: require(marker in content, f"{description} is missing {marker!r}") + if target.endswith("linux-gnu"): + for marker in (b'id="bundled-oss-cpp-sdk"', b'id="bundled-curl-8.13.0"'): + require(marker in content, f"{description} is missing {marker!r}") def parse_metadata(content: bytes, description: str) -> Message: @@ -466,6 +469,16 @@ def verify_wheel(path: Path) -> tuple[str, str]: name for name in names if name != native_member and is_native_library(name) ] if target.endswith("linux-gnu"): + bridge = "pypaimon_rust/liboss_cpp_bridge.so" + require( + bridge in extra_native_members, + f"wheel is missing OSS C++ bridge: {path.name}", + ) + with archive.open(bridge) as bridge_file: + verify_native_header( + bridge_file.read(65536), target, f"{path.name}:{bridge}" + ) + extra_native_members.remove(bridge) openssl_version = ( b"OpenSSL 1.1.1k FIPS" if target.startswith("x86_64") diff --git a/third-party-licenses/README.md b/third-party-licenses/README.md index 449c9aa3e..fd6ec0886 100644 --- a/third-party-licenses/README.md +++ b/third-party-licenses/README.md @@ -30,6 +30,10 @@ The XZ Utils 5.8.3 license is read from the locked `liblzma-sys` crate at `c62e0df1f9dcc2cc1e014711c5aa4561ae260538`. It covers `jieba-rs` 0.10.3 and `jieba-macros` 0.10.3. +Linux Python wheels also bundle OSS C++ SDK v2 at +`c13a82038b0696f99c32f4e86b623ffcc9e62140` and curl 8.13.0. +Their notices and curl license are copied here from those pinned sources. + Target-specific Python and Go reports are generated in their binary build jobs. They are intentionally not committed or included in the ASF source archive. @@ -38,3 +42,5 @@ Sources: - - - +- +- diff --git a/third-party-licenses/curl-8.13.0.COPYING b/third-party-licenses/curl-8.13.0.COPYING new file mode 100644 index 000000000..3fa85ebb6 --- /dev/null +++ b/third-party-licenses/curl-8.13.0.COPYING @@ -0,0 +1,22 @@ +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1996 - 2025, Daniel Stenberg, , and many +contributors, see the THANKS file. + +All rights reserved. + +Permission to use, copy, modify, and distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright +notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall not +be used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization of the copyright holder. diff --git a/third-party-licenses/oss-cpp-sdk-v2.THIRD_PARTY_NOTICES b/third-party-licenses/oss-cpp-sdk-v2.THIRD_PARTY_NOTICES new file mode 100644 index 000000000..ddb6c7b94 --- /dev/null +++ b/third-party-licenses/oss-cpp-sdk-v2.THIRD_PARTY_NOTICES @@ -0,0 +1,87 @@ +This file contains the licenses for third-party libraries distributed +with this software. + +============================================================================== +tinyxml2 +Source: sdk/src/thirdparty/tinyxml2/ +License: Zlib +============================================================================== + +Original code by Lee Thomason (www.grinninglizard.com) + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must +not claim that you wrote the original software. If you use this +software in a product, an acknowledgment in the product documentation +would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and +must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source +distribution. + +============================================================================== +base64 +Source: sdk/src/thirdparty/base64/ +License: Boost Software License 1.0 +============================================================================== + +Portions from https://github.com/boostorg/beast/core/detail/base64.h + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +============================================================================== +hash (MD5, SHA1, SHA256, HMAC) +Source: sdk/src/thirdparty/hash/ +License: Zlib +============================================================================== + +Copyright (c) 2014,2015 Stephan Brumme. All rights reserved. +see http://create.stephan-brumme.com/disclaimer.html + +This software is provided 'as-is', without any express or implied warranty. +In no event will the author be held liable for any damages arising from the +use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not +claim that you wrote the original software. + +2. If you use this software in a product, an acknowledgment in the product +documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must not be +misrepresented as being the original software.