diff --git a/.dockerignore b/.dockerignore index 28d48eb939..e8b92e609a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ # Pre-extracted release dirs / archives apache-hugegraph-*/ +**/apache-hugegraph-*/ **/*.tar **/*.tar.gz **/*.zip diff --git a/.github/configs/settings.xml b/.github/configs/settings.xml index 45c87139a9..6cc7dc435c 100644 --- a/.github/configs/settings.xml +++ b/.github/configs/settings.xml @@ -27,6 +27,13 @@ staged-releases https://repository.apache.org/content/groups/staging/ + + github + https://maven.pkg.github.com/hugegraph/toplingdb + + true + + diff --git a/.github/workflows/check-dependencies.yml b/.github/workflows/check-dependencies.yml index 23efa75ddb..a486dd56c2 100644 --- a/.github/workflows/check-dependencies.yml +++ b/.github/workflows/check-dependencies.yml @@ -25,8 +25,10 @@ jobs: dependency-check: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. SCRIPT_DEPENDENCY: install-dist/scripts/dependency + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Checkout source uses: actions/checkout@v4 diff --git a/.github/workflows/cluster-test-ci.yml b/.github/workflows/cluster-test-ci.yml index c2972cb9f9..8c8b03b4a1 100644 --- a/.github/workflows/cluster-test-ci.yml +++ b/.github/workflows/cluster-test-ci.yml @@ -30,7 +30,9 @@ jobs: cluster-test: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d66dc8cee9..52f3d2a93e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,7 +12,9 @@ on: jobs: analyze: env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} name: Analyze runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/commons-ci.yml b/.github/workflows/commons-ci.yml index fd01df1825..5919cb9499 100644 --- a/.github/workflows/commons-ci.yml +++ b/.github/workflows/commons-ci.yml @@ -9,16 +9,27 @@ on: - 'test-*' pull_request: +permissions: + contents: read + packages: read + +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: build-commons: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} strategy: fail-fast: false matrix: - JAVA_VERSION: ['11'] + JAVA_VERSION: [ '11' ] steps: - name: Install JDK ${{ matrix.JAVA_VERSION }} diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 97e2d37a41..cbb4d928b7 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -64,7 +64,9 @@ jobs: in_build && seen && /^FROM / { exit } - in_build { + # Compare Docker instructions only. Comments between stages may + # differ when a Dockerfile adds an optional build target. + in_build && $0 !~ /^[[:space:]]*(#.*)?$/ { print seen = 1 } @@ -89,7 +91,11 @@ jobs: run: | set -euo pipefail - docker buildx bake --file docker/bake.hcl --print > /tmp/hugegraph-bake.json + docker buildx bake --file docker/bake.hcl --print \ + > /tmp/hugegraph-bake.json + RUNTIME_VARIANT=topling \ + docker buildx bake --file docker/bake.hcl --print \ + > /tmp/hugegraph-bake-topling.json jq -e ' .group.default.targets == [ "build-cache", @@ -101,23 +107,145 @@ jobs: ([.target[] | .platforms] | all(. == ["linux/amd64", "linux/arm64"])) and .target["build-cache"].target == "build" and + .target.pd.target == "standard" and + .target.store.target == "standard" and + .target["server-standalone"].target == "standard" and + (.target["server-hstore"].target == null) and .target["build-cache"].output[0].type == "cacheonly" and + ([.target[] | .args.SOURCE_REPOSITORY] | + all(. == "https://github.com/apache/hugegraph")) and ([.target | to_entries[] | select(.key != "build-cache") | .value.output[0].type] | all(. == "docker")) ' /tmp/hugegraph-bake.json + jq -e ' + .target.pd.target == "topling" and + .target.store.target == "topling" and + .target["server-standalone"].target == "topling" and + (.target["server-hstore"].target == null) and + ([.target[] | .platforms] | + all(. == ["linux/amd64"])) + ' /tmp/hugegraph-bake-topling.json + + - name: Validate ToplingDB Compose parameters + env: + HUGEGRAPH_ADMIN_PASSWORD: ci-test-password + HG_PD_AUTH_SECRET_KEY: ci-test-pd-secret + HUGEGRAPH_SERVER_PULL_POLICY: always + HUGEGRAPH_PD_PULL_POLICY: always + HUGEGRAPH_STORE_PULL_POLICY: always + run: | + set -euo pipefail + + env \ + HUGEGRAPH_SERVER_IMAGE=local/hugegraph:topling \ + HUGEGRAPH_SERVER_VOLUME=server-topling-data \ + HG_SERVER_ROCKSDB_PROVIDER=topling \ + HG_SERVER_DATA_PATH=/hugegraph-server/topling-data \ + HG_SERVER_ENFORCE_PROVIDER_MARKER=true \ + docker compose -f docker/docker-compose.yml \ + config --format json > /tmp/topling-standalone.json + env \ + HUGEGRAPH_PD_IMAGE=local/hugegraph-pd:topling \ + HUGEGRAPH_PD_VOLUME=pd-topling-data \ + HG_PD_ROCKSDB_PROVIDER=topling \ + HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data \ + HG_PD_ENFORCE_PROVIDER_MARKER=true \ + HUGEGRAPH_STORE_IMAGE=local/hugegraph-store:topling \ + HUGEGRAPH_STORE_VOLUME=store-topling-data \ + HG_STORE_ROCKSDB_PROVIDER=topling \ + HG_STORE_DATA_PATH=/hugegraph-store/topling-storage \ + HG_STORE_ENFORCE_PROVIDER_MARKER=true \ + docker compose -f docker/docker-compose-hstore.yml \ + -f docker/docker-compose.dev.yml \ + config --format json > /tmp/topling-hstore.json + env \ + HUGEGRAPH_PD_IMAGE=local/hugegraph-pd:topling \ + HUGEGRAPH_PD0_VOLUME=hg-pd0-topling-data \ + HUGEGRAPH_PD1_VOLUME=hg-pd1-topling-data \ + HUGEGRAPH_PD2_VOLUME=hg-pd2-topling-data \ + HG_PD_ROCKSDB_PROVIDER=topling \ + HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data \ + HG_PD_ENFORCE_PROVIDER_MARKER=true \ + HUGEGRAPH_STORE_IMAGE=local/hugegraph-store:topling \ + HUGEGRAPH_STORE0_VOLUME=hg-store0-topling-data \ + HUGEGRAPH_STORE1_VOLUME=hg-store1-topling-data \ + HUGEGRAPH_STORE2_VOLUME=hg-store2-topling-data \ + HG_STORE_ROCKSDB_PROVIDER=topling \ + HG_STORE_DATA_PATH=/hugegraph-store/topling-storage \ + HG_STORE_ENFORCE_PROVIDER_MARKER=true \ + docker compose -f docker/docker-compose-3pd-3store-3server.yml \ + config --format json > /tmp/topling-3x3.json + + jq -e ' + .name == "hugegraph-standalone" and + .services.server.image == "local/hugegraph:topling" and + .services.server.pull_policy == "always" and + .services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == "topling" and + .services.server.environment.HG_SERVER_DATA_PATH == "/hugegraph-server/topling-data" and + .services.server.environment.HG_SERVER_ENFORCE_PROVIDER_MARKER == "true" and + (.services.server.volumes | + any(.source == "server-topling-data" and + .target == "/hugegraph-server/topling-data")) + ' /tmp/topling-standalone.json + + jq -e ' + .name == "hugegraph-hstore" and + .services.pd.image == "local/hugegraph-pd:topling" and + .services.store.image == "local/hugegraph-store:topling" and + .services.pd.environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + .services.store.environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + .services.server.environment.HG_SERVER_BACKEND == "hstore" and + (.services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == null) and + .services.server.image == "hugegraph/server:dev" and + .services.server.pull_policy == "always" and + .services.pd.pull_policy == "always" and + .services.store.pull_policy == "always" and + (.services.pd.ports | all(.target != 2012)) and + (.services.store.ports | all(.target != 2013)) and + (.services.pd.volumes | + any(.source == "pd-topling-data")) and + (.services.store.volumes | + any(.source == "store-topling-data")) + ' /tmp/topling-hstore.json + + jq -e ' + .name == "hugegraph-3x3" and + ([.services.pd0, .services.pd1, .services.pd2] | + all(.environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + .image == "local/hugegraph-pd:topling" and + .pull_policy == "always")) and + ([.services.store0, .services.store1, .services.store2] | + all(.environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + .image == "local/hugegraph-store:topling" and + .pull_policy == "always")) and + ([.services.server0, .services.server1, .services.server2] | + all(.environment.HG_SERVER_BACKEND == "hstore" and + (.environment.HG_SERVER_ROCKSDB_PROVIDER == null) and + .image == "hugegraph/server:latest" and + .pull_policy == "always")) and + ([.services.pd0, .services.pd1, .services.pd2, + .services.store0, .services.store1, .services.store2] | + all(.pull_policy == "always")) + ' /tmp/topling-3x3.json docker-build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - dockerfile: - - hugegraph-pd/Dockerfile - - hugegraph-store/Dockerfile - - hugegraph-server/Dockerfile - - hugegraph-server/Dockerfile-hstore + include: + - dockerfile: hugegraph-pd/Dockerfile + - dockerfile: hugegraph-pd/Dockerfile + target: topling + - dockerfile: hugegraph-store/Dockerfile + - dockerfile: hugegraph-store/Dockerfile + target: topling + - dockerfile: hugegraph-server/Dockerfile + - dockerfile: hugegraph-server/Dockerfile + target: topling + - dockerfile: hugegraph-server/Dockerfile-hstore steps: - name: Checkout @@ -125,18 +253,130 @@ jobs: - name: Build ${{ matrix.dockerfile }} run: | - IMAGE_ID=$(docker build -q --build-arg SOURCE_REVISION="$GITHUB_SHA" \ - -f ${{ matrix.dockerfile }} .) + TARGET_ARGS=() + if [[ -n "${{ matrix.target }}" ]]; then + TARGET_ARGS+=(--target "${{ matrix.target }}") + fi + IMAGE_ID=$(docker build -q "${TARGET_ARGS[@]}" \ + --build-arg SOURCE_REPOSITORY="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --build-arg SOURCE_REVISION="$GITHUB_SHA" \ + -f ${{ matrix.dockerfile }} .) echo "Built: $IMAGE_ID" echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" HC=$(docker inspect --format='{{json .Config.Healthcheck}}' "$IMAGE_ID") echo "Healthcheck: $HC" [[ "$HC" != "null" ]] || { echo "ERROR: HEALTHCHECK missing in ${{ matrix.dockerfile }}"; exit 1; } + SOURCE_LABEL=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.source"}}' "$IMAGE_ID") + REVISION_LABEL=$(docker inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' "$IMAGE_ID") + RUNTIME_LABEL=$(docker inspect --format='{{index .Config.Labels "org.apache.hugegraph.rocksdb-runtime"}}' "$IMAGE_ID") + [[ "$SOURCE_LABEL" == "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" ]] + [[ "$REVISION_LABEL" == "$GITHUB_SHA" ]] + if [[ "${{ matrix.dockerfile }}" == "hugegraph-server/Dockerfile-hstore" ]]; then + [[ "$RUNTIME_LABEL" == "hstore" ]] + elif [[ "${{ matrix.target }}" == "topling" ]]; then + [[ "$RUNTIME_LABEL" == "topling" ]] + else + [[ "$RUNTIME_LABEL" == "standard" ]] + fi - name: Test server entrypoint property mapping - if: matrix.dockerfile == 'hugegraph-server/Dockerfile' + if: matrix.dockerfile == 'hugegraph-server/Dockerfile' && matrix.target != 'topling' run: bash hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint.sh + - name: Verify ToplingDB image runtime + if: matrix.target == 'topling' + run: | + case "${{ matrix.dockerfile }}" in + hugegraph-pd/Dockerfile) + EXPECTED_ENV='HG_PD_ROCKSDB_PROVIDER=topling' + DATA_ENV_PREFIX='HG_PD_DATA_PATH=' + STANDARD_VOLUME='/hugegraph-pd/pd_data' + TOPLING_VOLUME='/hugegraph-pd/topling-pd-data' + ;; + hugegraph-store/Dockerfile) + EXPECTED_ENV='HG_STORE_ROCKSDB_PROVIDER=topling' + DATA_ENV_PREFIX='HG_STORE_DATA_PATH=' + STANDARD_VOLUME='/hugegraph-store/storage' + TOPLING_VOLUME='/hugegraph-store/topling-storage' + ;; + hugegraph-server/Dockerfile) + EXPECTED_ENV='HG_SERVER_ROCKSDB_PROVIDER=topling' + DATA_ENV_PREFIX='HG_SERVER_DATA_PATH=' + STANDARD_VOLUME='/hugegraph-server/rocksdb-data' + TOPLING_VOLUME='/hugegraph-server/topling-data' + ;; + *) + echo "ERROR: unsupported Topling Dockerfile" >&2 + exit 1 + ;; + esac + docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \ + "$IMAGE_ID" | grep -qx "$EXPECTED_ENV" + if docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \ + "$IMAGE_ID" | grep -q "^${DATA_ENV_PREFIX}"; then + echo "ERROR: data path must follow the selected provider" >&2 + exit 1 + fi + VOLUMES=$(docker inspect --format='{{json .Config.Volumes}}' \ + "$IMAGE_ID") + jq -e --arg standard "$STANDARD_VOLUME" --arg topling "$TOPLING_VOLUME" \ + 'has($standard) and has($topling)' <<<"$VOLUMES" + docker run --rm --entrypoint bash "$IMAGE_ID" -c ' + set -euo pipefail + test -n "$(find lib/topling -maxdepth 1 -name "rocksdbjni*.jar" -print -quit)" + test -r library/librocksdbjni-linux64.so + LDD_OUTPUT=$(ldd library/librocksdbjni-linux64.so) + echo "$LDD_OUTPUT" + if grep -q "not found" <<<"$LDD_OUTPUT"; then + exit 1 + fi + # This ABI probe bypasses the entrypoint. Provision the clean + # Server package defaults so the real pre-open guards can run. + if [ -f conf/rest-server.properties ]; then + mkdir -p rocksdb-data/data rocksdb-data/wal + fi + TOPLINGDB_ROCKSDB_PROVIDER=topling source bin/preload-topling.sh + test "$TOPLING_ACTIVE_NATIVE" = \ + "$PWD/library/librocksdbjni-linux64.so" + /bin/true + ' + + - name: Preserve standard image volume compatibility + if: matrix.target != 'topling' && matrix.dockerfile != 'hugegraph-server/Dockerfile-hstore' + run: | + case "${{ matrix.dockerfile }}" in + hugegraph-pd/Dockerfile) + DATA_ENV_PREFIX='HG_PD_DATA_PATH=' + LEGACY_VOLUME='/hugegraph-pd' + ;; + hugegraph-store/Dockerfile) + DATA_ENV_PREFIX='HG_STORE_DATA_PATH=' + LEGACY_VOLUME='/hugegraph-store' + ;; + hugegraph-server/Dockerfile) + DATA_ENV_PREFIX='HG_SERVER_DATA_PATH=' + LEGACY_VOLUME='/hugegraph-server' + ;; + esac + if docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \ + "$IMAGE_ID" | grep -q "^${DATA_ENV_PREFIX}"; then + echo "ERROR: data path must follow the selected provider" >&2 + exit 1 + fi + VOLUMES=$(docker inspect --format='{{json .Config.Volumes}}' \ + "$IMAGE_ID") + jq -e --arg legacy "$LEGACY_VOLUME" 'has($legacy)' <<<"$VOLUMES" + + - name: Verify HStore image has no local Topling runtime + if: matrix.dockerfile == 'hugegraph-server/Dockerfile-hstore' + run: | + docker run --rm --entrypoint bash "$IMAGE_ID" -c ' + set -euo pipefail + test -z "$(find lib -path "*/topling/rocksdbjni*.jar" \ + -print -quit)" + test ! -e library/librocksdbjni-linux64.so + ' + # The startup preflight needs a socket-table tool, and the base image # ships none of its own. Without one every start reports "unknown" and # a duplicate start is no longer refused, so assert the image can diff --git a/.github/workflows/licence-checker.yml b/.github/workflows/licence-checker.yml index a6e6990a64..e1b8ee9f93 100644 --- a/.github/workflows/licence-checker.yml +++ b/.github/workflows/licence-checker.yml @@ -11,7 +11,9 @@ jobs: check-license: runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 79217cfc58..bcdc0274b3 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -26,8 +26,154 @@ on: - '!**/AGENTS.md' - '!**/CLAUDE.md' +permissions: + contents: read + packages: read + # TODO: consider merge to one ci.yml file jobs: + distributed-rocksdb-runtime: + name: distributed-rocksdb-runtime (${{ matrix.check_name }}) + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + component: pd + provider: rocksdb + check_name: pd, rocksdb + experimental: false + - os: ubuntu-24.04 + component: pd + provider: topling + check_name: pd, topling + experimental: false + - os: ubuntu-24.04 + component: store + provider: rocksdb + check_name: store, rocksdb + experimental: false + - os: ubuntu-24.04 + component: store + provider: topling + check_name: store, topling + experimental: false + - os: ubuntu-26.04 + component: pd + provider: topling + check_name: pd, topling, ubuntu-26.04 + experimental: true + - os: ubuntu-26.04 + component: store + provider: topling + check_name: store, topling, ubuntu-26.04 + experimental: true + env: + TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Java 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'zulu' + + - name: Use staged maven repo settings + run: | + mkdir -p "$HOME/.m2" + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" + + - name: Package distributions + run: | + mvn clean package -Dmaven.test.skip=true -ntp + + - name: Select ${{ matrix.component }} ${{ matrix.provider }} distribution + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + if [ "${{ matrix.component }}" = "pd" ]; then + COMPONENT_DIR="hugegraph-pd/apache-hugegraph-pd-$VERSION" + if [ "${{ matrix.provider }}" = "rocksdb" ]; then + CONFIG_FILE="$COMPONENT_DIR/conf/application.yml" + sed -i 's/^# rocksdb:/rocksdb:/' "$CONFIG_FILE" + sed -i 's/^# provider: rocksdb/ provider: rocksdb/' \ + "$CONFIG_FILE" + fi + else + COMPONENT_DIR="hugegraph-store/apache-hugegraph-store-$VERSION" + if [ "${{ matrix.provider }}" = "rocksdb" ]; then + CONFIG_FILE="$COMPONENT_DIR/conf/application-pd.yml" + sed -i 's/^ # provider: topling/ provider: rocksdb/' \ + "$CONFIG_FILE" + fi + fi + if [ "${{ matrix.provider }}" = "topling" ]; then + touch "$COMPONENT_DIR/bin/pid" + if [ "${{ matrix.component }}" = "pd" ]; then + ln -s /missing/topling-ci-pd-data "$COMPONENT_DIR/pd_data" + else + touch "$COMPONENT_DIR/storage" + fi + install-dist/scripts/build-topling-distribution.sh \ + "${{ matrix.component }}" "$VERSION" + COMPONENT_DIR="$COMPONENT_DIR-topling" + fi + if [ "${{ matrix.component }}" = "pd" ]; then + STANDARD_COMPONENT_DIR="hugegraph-pd/apache-hugegraph-pd-$VERSION" + else + STANDARD_COMPONENT_DIR="hugegraph-store/apache-hugegraph-store-$VERSION" + fi + { + echo "VERSION=$VERSION" + echo "SERVER_DIR=$SERVER_DIR" + echo "STANDARD_COMPONENT_DIR=$STANDARD_COMPONENT_DIR" + echo "COMPONENT_DIR=$COMPONENT_DIR" + } >> "$GITHUB_ENV" + + - name: Verify clean ToplingDB distribution + if: matrix.provider == 'topling' + env: + TOPLING_EXPECT_DIRTY_STANDARD: 'true' + run: | + "$TRAVIS_DIR/test-topling-distribution.sh" \ + "${{ matrix.component }}" "$STANDARD_COMPONENT_DIR" "$COMPONENT_DIR" + + - name: Test standard RocksDB runtime + if: matrix.provider == 'rocksdb' + run: | + "$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + rocksdb "$COMPONENT_DIR" + + - name: Run Topling native lifecycle diagnostic + if: matrix.provider == 'topling' + run: | + "$TRAVIS_DIR/run-topling-native-diagnostic.sh" \ + "$RUNNER_TEMP/topling-native-diagnostic" \ + "$TRAVIS_DIR/test-rocksdb-runtime.sh" topling "$COMPONENT_DIR" + + - name: Upload Topling native diagnostic evidence + if: always() && matrix.provider == 'topling' + uses: actions/upload-artifact@v4 + with: + name: topling-native-${{ matrix.component }}-${{ matrix.os }} + path: ${{ runner.temp }}/topling-native-diagnostic/ + if-no-files-found: warn + + - name: Verify standard runtime packaging + if: matrix.component == 'pd' && matrix.provider == 'rocksdb' + run: | + "$TRAVIS_DIR/test-topling-runtime-packaging.sh" \ + "$SERVER_DIR" \ + "hugegraph-pd/apache-hugegraph-pd-$VERSION" \ + "hugegraph-store/apache-hugegraph-store-$VERSION" + struct: runs-on: ubuntu-latest env: @@ -86,11 +232,13 @@ jobs: runs-on: ubuntu-latest env: # TODO: avoid duplicated env setup in pd & store - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_FILE: hugegraph-pd/hg-pd-test/target/site/jacoco/jacoco.xml TEST_REPORT_DIR: hugegraph-pd/hg-pd-test/target/surefire-reports + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -117,10 +265,15 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Install deps + run: | + "${TRAVIS_DIR}/install-deps.sh" || exit 1 + - name: Package # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source "${TRAVIS_DIR}/install-rocksdb.sh" pd - name: Check source formatting run: | @@ -224,11 +377,13 @@ jobs: needs: struct runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. # TODO: remove outdated env TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_FILE: hugegraph-store/hg-store-test/target/site/jacoco/jacoco.xml TEST_REPORT_DIR: hugegraph-store/hg-store-test/target/surefire-reports + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -255,10 +410,15 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml mv -vf .github/configs/settings.xml $HOME/.m2/settings.xml + - name: Install deps + run: | + "${TRAVIS_DIR}/install-deps.sh" || exit 1 + - name: Package # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source "${TRAVIS_DIR}/install-rocksdb.sh" store - name: Check source formatting run: | @@ -368,11 +528,13 @@ jobs: needs: struct runs-on: ubuntu-latest env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: hstore RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') || startsWith(github.base_ref, 'release-') }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} steps: - name: Install JDK 11 @@ -393,6 +555,10 @@ jobs: with: fetch-depth: 2 + - name: Install deps + run: | + "${TRAVIS_DIR}/install-deps.sh" || exit 1 + - name: use staged maven repo settings if: ${{ env.USE_STAGE == 'true' }} run: | @@ -403,6 +569,9 @@ jobs: # todo remove --fail-at-end after test run: | mvn clean package -U -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp --fail-at-end + source "${TRAVIS_DIR}/install-rocksdb.sh" pd + source "${TRAVIS_DIR}/install-rocksdb.sh" store + source "${TRAVIS_DIR}/install-rocksdb.sh" hstore - name: Prepare env and service run: | diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index b5f5efc270..fdd5e3e5a1 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -9,6 +9,117 @@ on: pull_request: jobs: + server-rocksdb-runtime: + name: server-rocksdb-runtime (${{ matrix.check_name }}) + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + permissions: + contents: read + packages: read + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + provider: rocksdb + check_name: rocksdb + experimental: false + - os: ubuntu-24.04 + provider: topling + check_name: topling + experimental: false + - os: ubuntu-26.04 + provider: topling + check_name: topling, ubuntu-26.04 + experimental: true + env: + TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Java 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'zulu' + + - name: Use staged maven repo settings + run: | + mkdir -p "$HOME/.m2" + cp -vf .github/configs/settings.xml "$HOME/.m2/settings.xml" + + - name: Package Server + run: | + mvn clean package -pl hugegraph-server/hugegraph-dist -am \ + -Dmaven.test.skip=true -ntp + + - name: Select ${{ matrix.provider }} distribution + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR="hugegraph-server/apache-hugegraph-server-$VERSION" + if [ "${{ matrix.provider }}" = "topling" ]; then + touch "$SERVER_DIR/bin/pid" + mkdir -p "$SERVER_DIR/logs" + touch "$SERVER_DIR/logs/topling-ci-marker" + install-dist/scripts/build-topling-distribution.sh server "$VERSION" + SERVER_DIR="$SERVER_DIR-topling" + else + GRAPH_CONF="$SERVER_DIR/conf/graphs/hugegraph.properties" + sed -i '/^[#]*rocksdb\.provider=/d' "$GRAPH_CONF" + echo "rocksdb.provider=rocksdb" >> "$GRAPH_CONF" + fi + echo "STANDARD_SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION" \ + >> "$GITHUB_ENV" + echo "SERVER_DIR=$SERVER_DIR" >> "$GITHUB_ENV" + + - name: Verify clean ToplingDB distribution + if: matrix.provider == 'topling' + env: + TOPLING_EXPECT_DIRTY_STANDARD: 'true' + run: | + "$TRAVIS_DIR/test-topling-distribution.sh" \ + server "$STANDARD_SERVER_DIR" "$SERVER_DIR" + + - name: Test standard RocksDB runtime + if: matrix.provider == 'rocksdb' + run: | + source "$SERVER_DIR/bin/preload-topling.sh" + "$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + rocksdb "$SERVER_DIR" + + - name: Run Topling native lifecycle diagnostic + if: matrix.provider == 'topling' + run: | + "$TRAVIS_DIR/run-topling-native-diagnostic.sh" \ + "$RUNNER_TEMP/topling-native-diagnostic" \ + "$TRAVIS_DIR/test-rocksdb-runtime.sh" topling "$SERVER_DIR" + + - name: Upload Topling native diagnostic evidence + if: always() && matrix.provider == 'topling' + uses: actions/upload-artifact@v4 + with: + name: topling-native-server-${{ matrix.os }} + path: ${{ runner.temp }}/topling-native-diagnostic/ + if-no-files-found: warn + + topling-runtime-shell-test: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Test strict component-local runtime selection + run: | + hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-native-diagnostic.sh + hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh + hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh + hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-server-entrypoint-roots.sh + hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-provider-ownership.sh + wait-storage-shell-test: permissions: contents: read @@ -24,8 +135,11 @@ jobs: build-server: runs-on: ubuntu-24.04 + permissions: + contents: read + packages: read env: - USE_STAGE: 'false' # Whether to include the stage repository. + USE_STAGE: 'true' # Whether to include the stage repository. TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis REPORT_DIR: target/site/jacoco BACKEND: ${{ matrix.BACKEND }} @@ -35,6 +149,8 @@ jobs: TARGET_BRANCH_NAME: ${{ github.base_ref != '' && github.base_ref || github.ref_name }} RELEASE_BRANCH: ${{ startsWith(github.ref_name, 'release-') || startsWith(github.ref_name, 'test-') }} RAFT_MODE: ${{ startsWith(github.head_ref, 'test') || startsWith(github.head_ref, 'raft') }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTOR: ${{ github.actor }} strategy: fail-fast: false @@ -55,16 +171,16 @@ jobs: java-version: '8' distribution: 'zulu' - - name: Prepare backend environment - run: | - $TRAVIS_DIR/install-backend.sh $BACKEND && jps -l - - name: Install Java ${{ matrix.JAVA_VERSION }} uses: actions/setup-java@v4 with: java-version: ${{ matrix.JAVA_VERSION }} distribution: 'zulu' + - name: Install deps + run: | + ${TRAVIS_DIR}/install-deps.sh + - name: Cache Maven packages uses: actions/cache@v4 with: @@ -78,9 +194,14 @@ jobs: cp $HOME/.m2/settings.xml /tmp/settings.xml cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml - - name: Compile + - name: Package run: | - mvn clean compile -U -Dmaven.javadoc.skip=true -ntp + mvn clean package -Dmaven.test.skip=true -ntp + source "$TRAVIS_DIR/install-rocksdb.sh" server + + - name: Prepare backend environment + run: | + $TRAVIS_DIR/install-backend.sh $BACKEND && jps -l - name: Validate Docker integration if: ${{ env.BACKEND == 'rocksdb' }} @@ -132,6 +253,8 @@ jobs: - name: Run start-hugegraph.sh foreground mode tests if: ${{ env.BACKEND == 'rocksdb' && steps.server-preflight.outputs.can_run == 'true' }} run: | + mvn package -Dmaven.test.skip=true -pl hugegraph-server/hugegraph-dist -am -ntp + source "$TRAVIS_DIR/install-rocksdb.sh" server VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ $TRAVIS_DIR/test-start-hugegraph.sh $SERVER_DIR diff --git a/.goal-task/toplingdb-linux-closure/lessons.md b/.goal-task/toplingdb-linux-closure/lessons.md new file mode 100644 index 0000000000..6e476390b2 --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/lessons.md @@ -0,0 +1,57 @@ +# ToplingDB 操作经验 + +按当前操作选择分节，不默认全部加载。任务状态见 [state.md](state.md)，实测证据见对应环境记录。 +此处只保留可复用做法；具体实验时序和历史参数不重复搬入。 + +## 通用：构建、归因与身份 + +- 标准和 TP 同败不能证明问题早已存在。对比引入前源码、同配置服务及触发它的共用代码；auth 回归就是请求清理使共享 schema cache 变冷后暴露的漏删。 +- 实际 JNI 身份必须同时核对唯一 Java JAR 的 CodeSource、native maps/hash 及 adapter 来源。标签、环境变量和标准 JNI 下的 TP 分支测试都不能替代它。 +- 用干净完整源码构建。GitHub archive 可能因 export-ignore 缺少 install-dist；不复用旧 target/classpath。逐组件记录源码、参数、包/JAR 和实际镜像身份。 +- Maven 与 Surefire fork JVM 同时传专属 java.io.tmpdir；仅设置 MAVEN_OPTS 不够。单列 assumption skip、ignored、断言结果和 JVM 退出码。 +- compile 成功不证明 package/install 可用。Boot 可执行 JAR 不能直接作普通模块依赖：薄主 JAR 与 exec 附件分开，发行包选正确附件。 +- runtime 依赖更新要同步 known-dependencies、LICENSE/NOTICE；不要为迎合旧清单重新加入已移除的库。 +- 已关闭 native handle 从集合移除时按对象身份判断，不能调用 native-backed equals。独立 launcher 应正常关闭 TaskManager/EventHub；超时杀进程不算通过。 +- native JAR 校验和相同只证明字节身份；来源、固定依赖、CPU/ABI 基线和许可证是独立核查项。 + +依据：[Linux 身份与验证](linux.md)、[Mac 构建与 native 回归](mac.md)。 + +## 通用：数据与生命周期 + +- provider marker、目录或 CURRENT 存在不等于 backend 已初始化；由 init-store 检查真实表/CF，再执行图 API。 +- snapshot 验证需覆盖数据内容、失败重开和 pending 恢复并发。保存的 schema manager/builder 可绕过新入口，局部 gate 测试不证明完整缓存/全图协议安全。 +- 故障后保留 checkpoint、pending 与锁。只解除测试人工 hold 或注入物，并核对数据/marker/锁身份；不能删除恢复材料强行启动。 +- checkpoint 默认可能不含 WAL。故障 fixture 要使用同库真实 WAL，记录采样身份及校验，不宣称默认 checkpoint 有 WAL tail。 +- 同时开库锁与同时恢复 pending 是两个场景；进程被拒绝后仍可能 versions 200，必须检查实际数据库与图 API。 +- REST FINISHED 清理须配套 session 登记/末次关闭并发保护、auth 关联删除和冷缓存回归；同步请求通过不能外推异步、流式及停机排空。 +- system/primitive 的名称不代表真实标签范围。auth 用户可为 -27；级联删除只排除已证实的特殊 OLAP 类型，并验证无关关系保留。 +- 首次失败保留，重试成功不能覆盖它；正常退出、native 关闭、在途请求结果和重启后数据分别断言。 + +依据：[Linux 故障证据](linux.md#真实服务与故障证据)、[auth 归因](linux.md#2026-10-03-ci-归因与后续计划)。 + +## Linux / 容器 + +- 启动 suite 会清进程、端口和 cron，只能运行在专属容器。单宿主 kind/多进程不能证明物理多机 HA；Ready 不能证明已确认数据可见。 +- 单 DB bind mount 不能只用 FileStore 判定。读取 mountinfo 并处理转义；Docker/direct/init 都应在认领根目录前检查最终所有图的 data/WAL 根。当前预检依赖 util-linux mountpoint 2.37+，非挂载码为 32；纯 HStore 不要求它。预检后外部改变挂载仍是竞态边界。 +- TP 稀疏文件需区分逻辑长度与实际块数；避免盲目全量哈希，并注明未逐字节覆盖范围。root 容器的 ACL/属主变化可能让宿主故障注入失效。 +- 容器重启后重新读取随机发布端口。REST 若只绑定容器内回环或网络别名，宿主访问/默认 healthcheck 可能不适用；按实际地址验证。 +- TP 的 LD_PRELOAD 和 java.library.path 必须指向同一已核验 native，避免 fallback 再加载一份触发 factory abort。只有精确已知合成 CF 断言允许例外，其他 abort 必须失败；abort 后用新临时根复测。 +- 容器 CPU 配额影响 REST worker/503；记录有效阈值，改大资源后的成功不能覆盖原失败或作为性能对照。 +- snapshot 缺 MANIFEST、WAL ENOSPC、发布失败应分别触达目标阶段并保留内层错误；单库恢复成功不能写成整图恢复成功。 +- 长 Gremlin 响应可能 gzip；先正确解压，再分别核对 HTTP、停机和重启。Store 地址变化后最终重连成功不能证明旧故障已修复。 + +依据：[Linux 验收记录](linux.md)。原实验细节按本页下方历史链接取用。 + +## Mac + +- Docker CLI/socket 挂起但 OrbStack 正常时，可对单条命令移除大小写 HTTP_PROXY/HTTPS_PROXY/ALL_PROXY，curl 使用 --noproxy '*'；先检查本地 API，不改全局代理。 +- Mac amd64 容器的 JNI 结果须注明模拟架构及资源限制；不能外推性能或 Linux 服务器部署结论。 +- 假 runtime/marker spy 只证明调用路径。fixture 必须满足所有前置配置，只在目标阶段注入失败，并断言实际 bootstrap；非零退出本身不足。 +- Maven 未进入编译/测试时，先核 reactor 与父坐标，不能报告回归通过。 + +依据：[Mac 开发与实测](mac.md)。 + +## 需要追溯实验时 + +[精简前经验及证据索引](https://github.com/hugegraph/hugegraph/tree/4720da91b5a3c99b426c2ac035533ebc84d4e4cb/.goal-task/toplingdb-linux-closure) +保留原始触发条件、日志编号与旧失败。按具体问题取用，不作为默认上下文。 diff --git a/.goal-task/toplingdb-linux-closure/linux.md b/.goal-task/toplingdb-linux-closure/linux.md new file mode 100644 index 0000000000..cb4b15d60d --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/linux.md @@ -0,0 +1,309 @@ +# Linux 验收与恢复入口 + +## 当前范围与源码 + +更新于 2026-10-03。当前门禁仅覆盖 ToplingDB（TP）适配自身引入或放大的问题； +本 PR 新增共用代码的回归也必须修复。详细旧命令与首次失败在 +[精简前完整记录](https://github.com/hugegraph/hugegraph/blob/4720da91b5a3c99b426c2ac035533ebc84d4e4cb/.goal-task/toplingdb-linux-closure/linux-history-20261002.md)，旧门禁归属不覆盖本页。 +用户确认 #252 性能为可选项，#213 正式发布流程/发行链独立后置，均不阻塞当前任务。 +本轮先完成 TP 引入或放大问题的故障安全、真实服务验证及必要修复交付。 + +| 源码或交付 | 当前用途与状态 | +| --- | --- | +| 冻结 `9d797c7608e244f03436ce11294d9bd72aba4d2d` | Linux 基线验收使用该 SHA，用户确认其 38 项 CI 成功。 | +| 本地四项修复 | 挂载预检 `388ec8970`、任务查询 `54e5bebae`、session pool 末次关闭 `6d5893fd6`、REST FINISHED 清理 `e17f1b6d8` 已整合并推送。 | +| 此前发布 `94da2e8de5214e84f1a16d58e3124db60a96783d` | 已发布完成；旧认证/执行审批阻碍已解除。 | +| 合并 `ed6c295f0e599e5cd506a5d6aa54361b00a42081` | 保留 upstream `82034fb9f` 的 legacy scheduler 删除；三人只读审查、干净编译和 15 项定向回归通过。 | +| master `176fb56dd747ef0f60a126cf721aa12d17627c31` | 已同步删除 inactive RedirectFilter 的改动，保留本分支已完成修复。 | +| 代码同步 `de541b11d66429b5c3c7272062330192e20de5aa` | 双亲为 94da2e8de 和 176fb56dd；普通合并已发布。格式/干净编译 exit 0，34 项标准定向回归、真实 TP truncate 1 项通过，三人只读审查无阻塞。 | +| WAL 验证源码 `993ff6fe6401d3b612c21ea6d6d50fa0c75ab61f` | 完成服务级 WAL 发布故障和 pending 并发；该版本的 auth CI 回归由下一项修复。 | +| 最新代码 `3aa44152e8e13749d82ba58449b93254384ed0ed` | auth 级联修复已推送并同步本地；标准/真实 TP 回归和服务验证通过，验收文档另行提交。最新 PR head 与 CI 通过 gh 核对。 | +| 隔离 journal/gate 候选 | 留在冻结 SHA 的未提交工作树，未整合进 PR，不作为 PR 实现或实测通过证据。 | + +代码同步时 compare master...toplingdb 的 behind 为 0；2026-10-03 查询当前 PR 为 MERGEABLE。 +旧 993ff6fe 的检查为 33 成功、5 失败；最终 1d976571 的 38 项检查全部成功，详见下方归因记录。 +冻结 SHA 的 CI 成功不能替代最新源码验证。 +后续通过 `gh` 核对 PR 与 master；不直接调用 Git，不重排、force-push 或自动合并 PR。 + +## 问题归属依据 + +以 TP 引入前 `337dc865`、upstream `82034fb9f`、冻结 SHA 和上述已发布 PR head 作比较。 +`StandardHugeGraph` 的 create/resume、`AbstractBackendStoreProvider` 的逐 store 调用和 +`OpenedRocksDB.createCheckpoint()` 的创建机制在 TP 引入前已存在。`SchemaManager` 全文件在 TP 引入前、 +upstream 与冻结版本相同；`CachedSchemaTransaction` 的 upstream 与冻结版本相同。 + +源码确认恢复缺少缓存刷新、跨生命周期保护及全图协调的机制早已存在，应独立跟进。 +保存对象应拒绝旧 epoch 是未提交候选新增的要求；三项红测没有在接入前版本实测， +不能宣称历史版本已复现相同断言。依据在 `resume-20261002/attribution/sources.json` +和 `method-comparison.json`；标准与 TP 同败不能单独证明问题早已存在。 +#212 通用关停排空及 #248 旧 clear 可见性调查也独立跟进，保留其原始失败。 + +#249 的 WAL 扩展 `457295ac8` 是本 PR 新增共用代码：复制/发布失败、数据损坏保护、 +失败后 reopen 和相关锁仍是 TP 适配验收内容，不能将整个 issue 忽略。 +#252 后置且可选，不作为当前完成条件；不等待通用在线恢复改造。 + +## 验收清单 + +| 项目 | 已有结果 | 剩余 TP 门禁与解除条件 | +| --- | --- | --- | +| #250/#251/#253 配置与隔离 | direct/init、Docker、默认/自定义 graphs、额外图、provider 冲突及所有根已实测；全图挂载预检修复已发布。 | master 同步后按实际变更回归；数据身份或 provider 改变时重新验证打开前拒绝与数据不变。 | +| #254 adapter truncate | 真实 TP 多键 truncate 1/0，清空、CF 保留、重新写读与关闭通过；标准对照通过。 | 新增相关源码变化才重跑，标准 JNI 不替代 TP 结果。 | +| #255 native diagnostic | 三组件真实 probe 通过；精确已知合成 CF 断言按约定例外；服务关闭另有实测。 | 其他 native 错误仍失败；若发现 TP 独有真实 DB/CF 关闭错误，最小复现并修复。 | +| #249 WAL 扩展 | helper、s 库复制故障、开库锁已有证据；最新 std/TP 服务 g 成员最终发布故障、材料保留、同根冷启和 CRUD 均通过。 | pending 恢复并发的标准/真实 TP 对照已通过；复核损坏保护、适用源码及交付记录。未发现此路径的新生产回归。 | +| #213 JNI 发行链（独立后置） | 本机 ABI、CPU 与真实加载核查完成，输入字节身份固定。 | 正式发布材料另行跟进，不阻塞当前任务；本轮不发布正式 JNI。 | +| #252 性能（可选后置） | 标准/TP 三轮对照 0/3。已有并发 GET 仅为正确性/接纳检查。 | 不阻塞当前任务。另行决定执行时固定源码与配置，至少三轮并保留统计。 | + +所有通过都绑定下方源码与产物；新 master 同步和未提交候选必须分别验证。 + +## 2026-09-28 固定源码验收进度 + +原始证据根： +`/home/soc-baidu/.codex/validation-runtime/toplingdb-linux-closure/accept-9d797c7-20260928/evidence/`。 +数据库和临时输出在同任务的 `data/` 根；命令、退出码、计数、skip 和首次响应留在对应 evidence。 + +| 实测 | 实际计数或结果 | 证据索引 | +| --- | --- | --- | +| 干净标准构建 | `mvn clean install -DskipTests` exit 0；仅证明构建，不计测试。 | `01-clean-install.log` | +| 冻结标准 Core | 818/0 failure/0 error/42 skip，Maven exit 0。 | `02-*` | +| 标准 session/helper | 27+16=43 项，0 failure/error/skip。 | `03-*` | +| PD 定向 | IndexAPIClusterStateTest 2/0/0/0。 | `04-*` | +| 真实 TP helper | 27/0/0，唯一 TP JNI，实际 native maps 核对。 | `05-*`、`06-*`、`tp-classpath-source.txt` | +| 真实 TP adapter truncate | 1/0/0，包含清空、CF 保留、重新写读、再次 truncate。 | `07-*` | +| 配置/发行 fixture 与完整安全回归 | Server/PD/Store 配置、entrypoint、ownership、runtime packaging、标准/TP 包合同均 exit 0。 | `08-*` 至 `11-*` | +| 标准三组件 runtime lifecycle | Server、PD、Store 各 exit 0、phase complete。 | `12-standard-runtime-*.log` | +| TP diagnostic | 三组件前置 probe exit 0；合成 lifecycle 精确断言 exit 134，分类结果 known-cf-assertion。 | `13-tp-diagnostic-{server,pd,store}/` | +| 标准/TP 完整启动 | 专属容器中同 backend 各连续两轮 16 passed/0 failed。 | `26-*`、`27-*` | +| 整合修复后标准 API | Schema 1、Edge 5、Vertex 4，共 10/0/0/0，服务停机 0。 | `253-*` | +| 整合修复后标准 Core | 819/0 failure/0 error/42 skip，Maven exit 0；多出的 1 项为 task tx 回归。 | `258-standard-core/` | +| 发布合并源码定向 | clean compile exit 0；五类回归 15/0/0，JVM exit 0。 | `resume-20261002/remote-merge-targeted-tests-final.log` | + +Maven 和 Surefire JVM 均使用专属 `java.io.tmpdir`；真实 TP 测试替换唯一标准 JNI， +记录 RocksDB.class CodeSource 与 `/proc/self/maps`。构建跳测与计数差异均不能隐去。 +启动 suite 覆盖 init-store、daemon、前台 HTTP、monitor、SIGKILL 137、SIGTERM 143 传播； +目录或 marker 存在不代替 init-store 的真实表/CF 初始化。 + +## 产物与 runtime 身份 + +固定 TP 输入如下，适用于本轮 Linux x86_64，不套用于其他架构： + +- JAR SHA-256：`86eb1bd3d9f84ef0dddd3fe95c640a6f145ca2d5a26a5ba628f1f298a2031fae`。 +- native SHA-256：`c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。 +- 本机 ELF64 x86_64、glibc 2.43，CPU 支持 AVX2/BMI1/BMI2，ldd 无缺失。 + ELF 最高符号需求为 GLIBC_2.38、GLIBCXX_3.4.32、CXXABI_1.3.13；这只证明本机兼容。 + +Server、PD、Store 的标准/TP tar 均从干净冻结源码构建。镜像来源为独立干净 checkout； +SOURCE_REVISION 为完整冻结 SHA、SOURCE_URL 为仓库 URL，标签 accept-9d797c7-{std,tp}。 +独立构建和 linux/amd64 目标的完整参数、full digest 与 inspect 在 `14-image-*`、`15-image-*`。 +下表 SHA-256 缩写仅作索引；原始完整值保留在上述证据及历史记录。 + +| 组件镜像 | RepoDigest / amd64 imageID 前缀 | 实际 runtime 与隔离 | +| --- | --- | --- | +| PD std | `3e7ae66980e1` / `86c21a3588fd` | 标准 JNI；digest 为多架构 manifest。 | +| Store std | `79197a51369d` / 同 digest | 标准 JNI，无 TP JAR/native。 | +| standalone Server std | `7733edc06dda` / 同 digest | 标准 JNI，无 TP JAR/native。 | +| HStore Server std/TP | `51953abb919f` / 同 digest | 两标签相同 imageID；Server 本地不加载 TP，实际存储由 PD/Store 承担。 | +| PD TP | `31fa84ca7629` / 同 digest | PD 服务内 JAR/native maps 和哈希已核对。 | +| Store TP | `a0df11c186ec` / 同 digest | Store 服务内 JAR/native maps 和哈希已核对。 | +| standalone Server TP | `29874890b975` / 同 digest | Server 服务内 JAR/native maps 和哈希已核对。 | + +镜像文件来源见 `16-*`，standalone 服务映射见 `17-*`；三镜像 open/put/get/close probe +见 `66-*`，PD/Store 长驻服务映射后补于 `216-*`。文件存在或 provider 环境变量不替代加载证据。 + +整合 `e17f1b6d8` 的 TP Server 实际 imageID/digest： +`sha256:7379d24f6231eba72762593ee5d47964014e31f3b0d4ef5ec2e68ed24402ac42`。 +该镜像使用包含四项修复的源码；其服务身份、核心 JAR 来源和 maps 留 `242-*`。 + +发布合并 `ed6c295f0` 的 TP Server manifest/imageID： +`sha256:bf12f2d0dbbdeb4205b53c24ab700cf268e3ac89148e21e525b01345ef7d4a1b`。 +OCI config：`sha256:822ec5c80f9ae46537e9324eb014d0090e74a4f25036f98eeb7960309cd17947`。 +该镜像从无 target 的新源码上下文 `--no-cache --pull` 构建； +source label、实际服务 PID JAR/native 哈希在 `resume-20261002/remote-merge-image-identity.json` +及 `runtime-processes-hashes`，不继承冻结服务结果。 + +新的 master 同步使用 `gh api` 获取完整干净源码并三方合并；16 个路径中唯一冲突是 +ApplicationConfig import，保留 HugeFactory/FINISHED 清理并删除旧 redirect import/注册。 +格式未额外修改源码；六类标准定向测试 34/0/0、JVM exit 0（AccessLogFilterTest 19 项， +移除的 1 项仅验证已删除 feature 的注册顺序）。实际 TP truncate 1/0/0，JAR/native +哈希与上述输入相同，CodeSource 指向新隔离 reactor，native maps 指向核验文件。 +证据在 `resume-20261002/master-176fb56/`：三方源码、resolved.patch、merge-manifest、 +build/test commands、日志、退出码、native identity、三份 review 和 published merge SHA。 +以上是代码同步时的定向验证；随后 993ff6fe 新产物及服务实测见下方恢复执行记录。 +三组件镜像并非全部重建，本次结果不替代未覆盖的服务范围。 + +## 真实服务与故障证据 + +| 场景 | 有效实测与边界 | 原始证据 | +| --- | --- | --- | +| standalone 冻结 TP | 两顶点一边、Gremlin、首次同根重启通过；首次 SIGTERM 出现 db not closed/exit 137。 | `17-*`、`18-*`、`20-*`、`21-*` | +| 自定义 graphs 与额外图 | 两图实际 CF、data/WAL 全在专属父根，CRUD 与重启通过；冻结版本关闭仍失败。 | `28-*` 至 `32-*` | +| provider 反向冲突 | 标准/TP 指向对方根均 exit 1，在数据库打开前拒绝；路径/大小/mtime及小文件哈希一致。稀疏大文件未全字节校验。 | `24-*`、`25-*` | +| 挂载预检修复 | direct/init/Docker 在所有图开库前检查 m/g/s，修复 fixture 23 PASS；三人审查与真实拒绝验证通过。 | `77-*`、`78-*`、`99-*`、`106-*`、归档修复段落 | +| 1+1+1、1+3+3 | 冻结三组件 CRUD、重启与恢复实测；PD/Store 独立数据根。单宿主不证明物理多机 HA。 | `34-*` 至 `55-*` | +| 整合 TP 服务 | CRUD、20/20 GET、首次重启、两次 SIGTERM 0，无 native 关闭断言。 | `242-*` | +| 流量中 SIGTERM | 3977/3977 GET 为 200，服务退出 0，同根重启数据校验通过。 | `259-*` | +| 真实 SIGKILL | exit 137；新有效实验的同根首次图 GET 200、CRUD/Gremlin 通过，最终正常停机 0。 | `264-*`；`262-*` 首次请求失败保留。 | +| ed6c295f0 服务兼容性 | 16 CPU/4 GiB 下 20/20 GET、同根重启、两次 SIGTERM 0，实际 TP 身份吻合。 | `resume-20261002/merge-service-v2/` | +| snapshot 成功路径 | std/TP create/resume 200；重启后快照前数据可读、快照后数据 404。同进程缓存失败另列通用调查。 | `56-*` 至 `65-*` | +| checkpoint 校验拒绝 | 缺 MANIFEST 首次 resume 400、无 pending；原文件/hash恢复后 reopen、resume 与重启断言通过。 | `108-*` 至 `116-*` | +| WAL staging 复制故障 | std/TP 的 64 MiB WAL 根注入 96 MiB 日志，实际 ENOSPC；s pending 与全部 checkpoint 保留，移除仅人工源后 s 重试成功。 | `123-*` 至 `130-*` | +| 数据树复制/校验故障 | std/TP 注入 checkpoint 悬空链接，失败材料保留；移除仅测试链接后 reopen/verify 成功。未取到内层栈，只证明 s 成员路径。 | `243-*` 至 `246-*`、`251-*` | +| 同时开库锁 | std/TP 第二进程被 RecoveryLockException 拒绝，第一进程数据不变；未删除 lock 强开。不是 pending 恢复并发。 | `254-*`、`256-*` | + +WAL 发布失败、嵌套/独立 WAL 与内容校验已有 JNI helper 覆盖；最新 std/TP 服务级 +g 发布故障另获通过证据，具体合成故障和真实 WAL fixture 边界见下方恢复执行记录。 +s 成员重试后 g/m checkpoint 残留不能声称全图恢复完成；该原有逐库协调缺口归独立调查。 + +首次失败原样保留：启动容器缺 compiler/预建目录，脚本端口/断言错误，SIGKILL 过早 GET +连接重置，JNI 路径遗漏导致重复加载 exit 134，merge 服务 4 CPU 下 17×200/3×503。 +后者日志明确 maxWorkerThreads=8 的负载保护；16 CPU 后续成功不覆盖首轮失败。 +详见 [历史完整记录](https://github.com/hugegraph/hugegraph/blob/4720da91b5a3c99b426c2ac035533ebc84d4e4cb/.goal-task/toplingdb-linux-closure/linux-history-20261002.md)，私有 native 日志不直接公开环境内容。 + +## 独立调查与隔离候选 + +| 问题 | 事实与归属 | 下一步边界 | +| --- | --- | --- | +| 通用缓存/全图恢复 | std/TP snapshot_resume 后同进程仍读到快照后顶点，重启后均 404。 | 按独立缓存/恢复 issue 跟进；不作为 TP 性能的统一前提。 | +| saved schema/epoch | 未提交 journal 候选的 std/TP 三项各 3/3 失败，旧 manager 旁路、旧 builder 创建 id=2 且 fresh manager 回读。 | 只证明候选未满足新增要求，历史断言未经基线实测；完整图接入不并入当前 TP 适配。 | +| journal/gate/iterator 候选 | 隔离 backend TP 恢复→truncate和启动重放各 1/0；synthetic gate 10、iterator 9，共 19/0，三人审查和复审完成。 | 不证明真实 native iterator/整体安全；16 条未提交路径与数据保留。 | +| #212 关停排空 | 真实 TP CRUD 关闭问题经四项修复改善；长 Gremlin 在途 SIGTERM仍 HTTP 500。 | 通用请求排空独立；仍关注新的 TP 独有 native 错误。 | +| #248 clear 可见性 | 冻结 std/TP、按 ID/属性、第二 Server、PD 先重启均未复现旧失败。 | 旧 a35ebeb17/schema ID 2/部署差异与首次失败保留，不自动关 issue。 | +| 路由/健康语义 | 单 Store 故障边扫描 500、PD 切主失败窗口、开库失败却 versions 200。 | 标准对照与通用平台归因在 todo.md；不声称零中断 HA。 | + +候选路径：`/home/soc-baidu/.codex/worktrees/topling-graph-journal/hugegraph`。 +本轮候选实测、三人报告、source manifest 和 retained data inventory 在 `resume-20261002/`； +失败后哈希无前置对照时只作留证，不能声称数据前后字节不变。 +暂停扩展前另改三文件加入恢复后回调与材料保留测试，仅静态检查，未编译/运行/审查； +这些修改同样留在隔离候选，不计本 PR 修复或通过。 + +## JNI 来源与正式发布 + +[#213](https://github.com/hugegraph/hugegraph/issues/213) 独立后置，不作为当前任务完成条件；正式发行前仍需补齐材料。JAR 对应 producer +构建提交 `31afa28f3d31606c1d5769a42a96ecd93420e8bc`、Ubuntu 24.04/GCC 13；资产 SHA-1 +相同仅证明字节身份。workflow 动态取得 SidePlugin、shallow checkout、deploy-file， +缺少完整不可变源码闭包、attestation 和明确 CPU 下限。 +2026-10-02 比较 fix-jni-ci `a46a3dfe4687f7744f0f5291a182f0d5378faae0` workflow 与旧构建版本， +文件逐字节相同；见 `resume-20261002/producer-*` 与 `252-producer-release-recheck/`。 +producer POM 列 Apache-2.0/GPLv2、包页标 GPLv2，本地 release LICENSE 列 Apache 2.0， +JAR 未发现 LICENSE/NOTICE；需要 producer/发行审查，不自行发布正式产物。 + +## 执行环境与下一动作 + +### 2026-10-02 恢复后的实际执行 + +用户已手动恢复 goal，工具核对为 active。本地 state/linux/todo 已按最新约定更新： +性能可选，正式发行链独立后置；当前执行 #249 故障安全与最新源码服务验证。 + +新验收产物来自 `993ff6fe6401d3b612c21ea6d6d50fa0c75ab61f`：按远端完整 tree 核对 +2723 个 tracked 文件，在新目录 clean install exit 0，Server/PD/Store 标准与 TP 六包均构建成功。 +两个独立 Server 测试镜像实际 imageID/manifest digest 为：标准 `sha256:352d00c1827e9d7f1f39460824a3333f2ce7b169be8b303cf5406c7955db1390`， +TP `sha256:bfde52672b061de1047e221435da50f5de22e042ccb8d44d92cbda2d4dd4cb76`。 +完整 build 参数、包哈希、源码清单、image inspect/metadata 留 +`resume-20261002/wal-publication-993ff6fe/`。本次没有重建 PD/Store 镜像或声称其新服务验收通过。 + +首次源码归档缺 export-ignore 文件导致 TP 脚本 exit 127；完整 tree 补齐后从另一新目录重建。 +默认 Docker 构建器访问 registry 连接重置，改用任务独立构建器后两镜像成功；构建器已停止。 +标准服务测试先遇 readiness 的 ConnectionResetError、再遇宿主机写 root-owned checkpoint 的权限错误， +均保留日志，不计 TP 回归。修正仅操作专属容器内 fixture；生产源码未改。 + +test-only agent 的三项 synthetic seam 检查通过：retire 不误注入、最终 publish 仅一次失败、 +未 armed 时重试可正常 move。标准真实服务命中最终 WAL publish 故障（HTTP 400）， +g pending 与 checkpoint 字节保留；脚本曾因要求日志包含异常全文而退出，首次退出保留。 +随后在同一保留数据根完成独立 recovery driver：不删 marker/lock、不带故障钩子， +pending 自动清理、g checkpoint 在成功重开后消费；首次 baseline 200、快照后顶点 404， +完整 smoke CRUD/Gremlin 校验通过，故障后及恢复后的关闭均 exit 0。 +标准用例通过证据在 service-rocksdb-v3/result.json 和各首次响应；首次脚本失败不覆盖。 +真实 TP 对照 service-topling-v5 也通过：同一最终发布故障 HTTP 400、g pending/全部 +checkpoint 文件哈希不变、不删 marker/lock 的冷启恢复、首次 baseline 200/快照后 404、 +完整 CRUD/Gremlin 验证及关闭 0/0。实际 JAR/native 的固定哈希与本轮身份匹配。 +若普通 checkpoint 无 WAL，fixture 明确添加来自本库、快照前稳定采样的真实 WAL 副本， +只用于触达发布分支；不造坏记录，也不声称默认 checkpoint 自然含 WAL tail。 + +真实 WAL fixture 另验证物理记录 CRC32C 与完整 WriteBatch 头；这不是完整 native ldb dump。 +runtime 核对标准 JNI JAR/ELF 哈希及 TP 固定身份；test-only agent 源码/JAR另留证。 +pending 并发 std/TP 的 fresh p2 用例均通过：恢复 owner 在 g 最终 publish 前保持锁， +另一个独立 JVM 直接调用 g 的 public sessions constructor，实际 JNI/adapter 身份匹配， +以 RecoveryLockException.isContention 拒绝；g 数据、checkpoint、pending 的 typed tree/字节， +以及锁文件 dev/inode/字节均未变。解除 test hold 后 owner 完成恢复、首次 baseline 200/after 404， +CRUD/Gremlin 和关闭 0/0。证据在 `resume-20261002/pending-concurrency-993ff6fe/`。 +test harness 独立审查及 delta 确认 fresh-only、唯一 JNI classpath 与取证修正；生产源码未改。 +TP p1 竞争 JVM 曾用标准 Java JAR+TP native，被身份断言拒绝；人工 hold 中清理 exit 137、未 OOM， +首次失败保留，不计 TP 通过。p2 两个 provider 才是完整有效对照；只证明 g 成员，不证明全图原子性。 + +较早 CI 快照为 33 成功、3 失败、2 执行中（`resume-20261002/ci-after-wal.json`）； +2026-10-03 的最终结果与当前计划见下一节，不能沿用该中间快照判断任务完成。 + +### 2026-10-03 CI 归因与后续计划 + +原 Linux goal 已按用户要求暂停。修复提交 `3aa44152e8e13749d82ba58449b93254384ed0ed` +已从隔离分支 `toplingdb-auth-cascade-20261003` 快进至原 toplingdb、PR #179 及本地 checkout； +父提交为原 PR head 993ff6fe，未重排或 force-push。 +两个远端 master 在本次核对均为 176fb56dd，未继续前进。用户最新决定取消 7% 保留限制，允许用完剩余周额度。 + +原 head 993ff6fe 的 38 项 CI 为 33 成功、5 失败，五个失败 job 均在 +`ProjectApiTest.testList` 删除后重建 `test_project46` 时返回 400，提示旧 access 已存在。 +每个 API suite 161 项、1 failure、0 error；HBase skip 13，其余 skip 14。 +最终快照和原始 job 日志在 `resume-20261002/ci-current-failures/`。 + +下表来自相同认证、graph fixture、8 CPU/4 GiB、新数据根及标准 JNI 的服务对照。 +Server PID 的 JAR/native 已逐一核验，删除均 204、服务关闭均 0；不能只以 auth 源码未改归为旧问题。 + +| 对照 | 同名创建结果 | 证据路径（相对 ci-current-failures/） | +| --- | --- | --- | +| head 993ff6fe | 201 → 400。 | `auth-sequence/head-std1/` | +| 固定 master 176fb56dd | 三轮均 201。 | `auth-sequence/master-std1/`；完整干净源码/构建另见 `master-source/`、`master-build/`。 | +| 同 head，仅测试禁用 FINISHED 清理 | 三轮均 201。 | `auth-sequence/head-control-std1/`；仅作归因控制。 | + +本 PR 的 FINISHED 清理销毁请求事务，schema 事务关闭会清共享缓存；隐藏系统 edge label +在冷枚举中不可见，导致删除的 `existsLinkLabel` 优化跳过关联边。auth entity/relationship +经 systemTransaction() 进入 SysTransaction，继承 GraphTransaction 的级联实现。 +该问题是本 PR 引入或放大的共享代码回归，详见 `schema-cascade-attribution.txt`。 + +最终修复保留 FINISHED 清理：系统顶点在冷缓存下也扫描关联边，只对实际 OLAP 标签保留原路径。 +不能用 `primitive()` 的 -32..-1 范围排除，因为动态系统标签从 -8 开始，实际用户标签为 -27。 +初版遗漏用户带关系删除的红测为 1 项/1 failure，残留 1 条 belong；最初项目红测为 +1 项/2 failures（主体残留 2 条 access，@After 检测残留总数 3）。首次失败均保留。 + +| 最终修复验证 | 实际结果 | 证据（auth-cascade-fix/ 下） | +| --- | --- | --- | +| 三名独立只读复审 | v4 的三个文件哈希匹配，均无 P0/P1；旧 v3 范围判断已被新红测推翻，不沿用其结论。 | `review-*-v4.txt`、`candidate-manifest.json` | +| 格式、干净编译、Server/test install | exit 0；Maven/JVM 均用专属临时目录。 | `build-commands-v4-retry.json` | +| 标准 Auth 与边界 | AuthTest 53、OLAP 保护 1、普通顶点删边 1，共 55；failure/ignored/assumption 均 0，JVM exit 0。 | `green-standard-v4.log` | +| 真实 TP 关键回归 | 项目、用户带关系、用户无关系、OLAP、普通顶点删除，共 5；failure/ignored/assumption 均 0，JVM exit 0。 | `green-real-tp-v4.log` | +| 完整标准 API suite | 161 项、0 failure、0 ignored、14 assumption skip；13 项 GraphSpace 需要 HStore，1 项 Gremlin clear 需要共享存储。服务 exit 0。 | `../auth-sequence/fix-api-suite-std2/` | +| 小批次级联提交 | 阈值 2，标准/真实 TP 各 1 项通过，三类失败/skip 计数均 0；原有分批提交异常传播问题仍独立。 | `lowbatch-{standard,tp}.log` | +| 真实认证服务 | 标准/TP 均通过：三轮项目删除重建、用户带关系删除和同名重建不继承旧 membership、无关关系保留、图 CRUD、首次同根重启 GET 200，各两次正常关闭 0/0。 | `../auth-sequence/fix-v4-{std,tp}/` | + +最终标准/TP Server 包来自该修复源码；完整 2723 个 tracked blob 与 commit tree 核对。 +两个独立镜像均 --no-cache --pull 构建：标准 manifest/imageID +`sha256:1b02774f40e8ffe36577fe93087a700b74bcc54596cb9aae3eca699b85268a0a`，TP +`sha256:92d4adcf751beba7a02c32afb98d3189540d0e00ce561d6079820e62d62b187f`。 +metadata 的 configDigest 为 null，未补造。包哈希、命令与 inspect 在 `auth-cascade-fix/`。 +实验 JAR/native 身份仍采用本页固定输入，另按首次与重启 Server PID 验证实际加载。 + +无效前置与较早候选仍保留：认证未配置、原 agent 反射被安全模式过滤、手工 classpath +混入 Struct 同名类、生成的旧 TP 发行目录触发 RAT、完整 API 首轮客户端不能访问容器内 Arthas 8561。 +API 改为在同一专属容器运行客户端后整套通过；首轮 161/1 failure/14 skip 保留。它们不计生产回归通过；生成目录已移出 +源码并保留。被替代的 43b88c1 候选、旧项目通过结果及 metadata 留在证据中，不代表最终用户边界通过。 + +冻结→候选的 PD/Store/Struct/Commons、POM/native/assembly 无变化，依据为 +`auth-cascade-fix/component-source-delta-audit.txt`、`final-component-source-delta.json` 及完整 tree JSON。最终修复只含 Server 的 +GraphTransaction 与两份测试。冻结 PD/Store 组件证据可复用；新 Server 独立验证与冻结 +分布式验证分别列明，不把两个新 Server 镜像算成新全栈部署。 + +1. 已发布代码 3aa44152，标准/TP 实际运行 core/API JAR 哈希与最终构建包一致，取证见 `final-runtime-verification.json`。 +2. 验收文档与 #249/#240 进展已发布；2026-10-03 交接核对确认 **1d976571 的 38 项 CI 全部成功**。先前排队快照只作历史记录。 +3. 原 goal 保持用户指定的暂停状态；新会话按 [拆分交接](pr-split-handoff.md) 在 org 建分支并先向 org master 提交实测，最终向 Apache master 提交合入。完整分支的通过不替代子 PR 及新 base 的验证。 + +#252 性能可选、#213 正式发行独立后置，均不占上述必需顺序。通用缓存/全图恢复问题按 todo.md +独立跟进；本轮不把隔离 journal/gate 候选合入 PR。 + +主机首次核对为 Linux x86_64、i9-13900KS、32 逻辑 CPU、123 GiB RAM；Java 11.0.32.1、 +Maven 3.9.12、Docker 29.1.3。历史服务清理回收约 84 GB；kind/BuildKit 已停止保留数据， +较早看到另一个任务的 hg-pr221 集群时未操作它；本轮最终核对无运行容器。 +本任务 auth 容器已逐个清理，独立构建器已停止；资源值仍按执行时重新核对。 +历史 `/home/soc-baidu/.codex/worktrees/f29e/hugegraph` 的 WAL/channel 补丁仍不复用。 + +每项证据须记录源码、构建参数、实际 imageID/digest、JAR/native、命令/退出码、计数/skip、 +数据断言和未覆盖边界。全部重任务串行，临时目录同时传 Maven/Surefire；启动清理只在专属容器。 +代码与文档分别发布；最新远端 head 通过 gh 核对。本页保留适用版本，旧流水账移至历史档案。 + diff --git a/.goal-task/toplingdb-linux-closure/mac.md b/.goal-task/toplingdb-linux-closure/mac.md new file mode 100644 index 0000000000..d9bda1c6fa --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/mac.md @@ -0,0 +1,237 @@ +# Mac 开发与恢复入口 + +与 [state.md](state.md) 一起读取即可恢复本机工作。本机 Linux 容器的核心实测记在这里， +远端 Linux 服务器验收只在 [linux.md](linux.md) 维护。 + +## 基线与下一动作 + +本机执行基线 `7c7519b79c445c8e3c03a510f9c6aa7484fc0103`，工作目录 +`/Users/zhu/.codex/worktrees/topling-local-fixes/hugegraph-server`，detached HEAD。 +远端 hugegraph 为 https://github.com/hugegraph/hugegraph.git，推送目标 `HEAD:toplingdb`。 +主 checkout 的无关文件及 Linux 历史未提交补丁未搬入本轮。 +恢复时重新核对 HEAD、工作区和远端；本轮最终提交见下方交付记录。 + +2026-09-27 更正：上一轮已交付本机定向验证，但整体 CI 门禁未收口。 +完整 CI 随后发现恢复锁相关的标准多盘回归，以及本 PR 新增 PD 测试的 package/install 编译失败。 +第一、二轮补救已交付；第二轮 head `89979877b` 的 CI 有 37 成功、1 失败。 +第三轮已修复真实启动 fixture 的 backend 初始化误判，并完成完整启动顺序与同 backend 复跑。 +不能把前轮 goal 的 complete 标记当作当前代码已具备合入条件。 +当前下一动作是核对第三轮提交 `8d92909b78f05e762e3e741a0949d6d33dac9374` 的推送结果及新 head 的完整相关远端 CI； +Linux 可先按已交付 SHA 验证独立项目，但须在新提交后复验受影响场景。不自动合并 PR。 + +## 开发清单与验收 + +| Issue | 本轮行为与证据 | 开发状态 | +| --- | --- | --- | +| [#250](https://github.com/hugegraph/hugegraph/issues/250) | 图配置决定有效 provider，环境双向冲突、多图混用明确失败；缺省保持 RocksDB | 实现及定向回归完成，最终交付见下方 | +| [#251](https://github.com/hugegraph/hugegraph/issues/251) | 读取实际 REST graphs，路径与 Java 启动工作目录一致；相对、绝对、空格、缺失目录、backend 大小写均有 fixture | 实现及定向回归完成，最终交付见下方 | +| [#253](https://github.com/hugegraph/hugegraph/issues/253) | direct/init 与 Docker 在 DB 打开前检查全部 data/WAL 根；标记原子认领、并发复查，拒绝冲突、符号链接、根目录和非空 data_disks | 实现及定向回归完成，最终交付见下方 | +| [#254](https://github.com/hugegraph/hugegraph/issues/254) | 实际 adapter 覆盖空、单 key、乱序多 key、空/二进制/无符号边界、多 CF、清空后读写；标准重建 CF 与 TP 范围删除分别断言 | 标准及真实 TP adapter 回归完成，原问题为覆盖不足，未据此认定历史数据损坏 | +| [#255](https://github.com/hugegraph/hugegraph/issues/255) | 独立前置 probe 必须通过；仅 cf-lifecycle + 134 + 精确已知断言且无其他错误可例外，取消无条件 continue-on-error | fixtures、真实 TP 分类完成；已知合成 CF 断言仍存在，#212 未关闭 | +| [#249](https://github.com/hugegraph/hugegraph/issues/249) | 恢复前持久化 pending、保留原 checkpoint，校验安装，正常 open 先重试未完成恢复；锁贯穿 DB 生命周期与重开，拒绝单 DB 挂载 | 标准与 TP 成功/故障回归完成；不声明断电持久性或非协作旧进程安全 | + +配置解析不能等价支持的 includes、续行、转义 key、重复/插值相关值明确拒绝，避免脚本与 Java 分叉。 +标准 RocksDB 的既有非空无标记目录保留兼容；新空根会认领标准 marker，TP 仍要求明确隔离且已存在的根。 +挂载父 data 根受支持；单 DB 卷挂载在打开前拒绝，Linux 还拒绝同文件系统 bind mount。 +恢复源只在 native 重开成功后尝试清理；异常保留源与 pending，不能删除 guard 强行启动。 + +## 最小验证与审查 + +以下为上一轮 `cf25a438a50f93f96efce74b9e4b0cacce4fac91` 所包含代码的历史定向证据， +不代表完整 CI 已通过，也不继承为本次补救代码已通过。证据目录 `/tmp/topling-local-20260927`，原日志不提交。 + +| 验证 | 结果与边界 | 日志 | +| --- | --- | --- | +| root `mvn -o editorconfig:format` / `mvn -o clean compile -Dmaven.javadoc.skip=true -ntp` | 格式与完整 reactor 编译通过；离线使用已安装依赖 | format-delivery.log / compile-delivery.log | +| Mac 标准 JNI 定向测试 | RocksDBSessionsTest 16 + RocksDBSnapshotRestoreTest 24，40 项无失败/错误/skip | java-tests-delivery.log | +| Linux arm64 shell | selector 31、roots 16、ownership 8，55 个 PASS；真实 provider verifier，runtime/启动用隔离 fixture | linux-shell-final.log | +| native diagnostic fixtures | 27 个 shell 场景及 2 个 Java 清理断言，29 个 PASS；覆盖混合错误、大小写、错误状态/阶段、缺前置、真实 C++ 表达式 | diagnostic-cpp-final.log | +| Mac 标准 runtime | probe 与实际 CF lifecycle 通过；修复已关闭 handle 的 equals 调用后复跑 | smoke-probe.log / smoke-lifecycle-fixed.log | +| TP adapter truncate / independent WAL | 各 1 项通过，实际加载 TP JAR 与 native | tp-adapter-truncate.log / tp-wal-separate.log | +| TP WAL helper | 24 项无失败/skip，覆盖故障安装、短复制/同长损坏、缺源、启动重试、独立/嵌套 WAL、符号链接、进程锁、重复恢复 | tp-wal-faults-delivery.log | +| TP native 分类 | probe 读写/遍历/关库成功；合成 CF phase 实际 134，分类 known-cf-assertion，wrapper 退出 0 | tp-native-diagnostic-verified-fixed.log 及同名目录 | +| 挂载 admission | TP 的单 DB host mount、同文件系统 bind、带空格 mount 均在变更前拒绝；父根双别名共享锁，关库后可重开；Mac HFS+ 卷在数据/锁写入前拒绝 | bind-guard-all-final.log / mac-mount-guard-final-fixed.log | + +原有 Server/Docker、PD/Store entrypoint fixtures 通过； +entrypoints-linux-final.log 保存 Linux entrypoint 最新结果。 +新 helper/fixture shellcheck 无新发现，actionlint 对比基线无新增发现。 +GitHub 本次代码 head 的 Commons/dependency-check 等失败日志指出 TP 图片索引三文件缺末尾 LF； +文档批次补 LF，`mvn -o editorconfig:check -ntp` 通过（editorconfig-check-delivery.log）。 +741c 的 Commons 检查仍失败，此时 LF 修复尚未推送；须核对本次文档批次的新 head，不能继承旧 run。 +旧交付将 hg-pd-test 编译失败当作本轮 diff 外的独立门禁，这个归属判断不完整： +IndexAPIClusterStateTest 由本 PR 的 `327737f16` 引入，虽早于上轮 goal，仍由本分支负责。 +本次归因和修复见下方 CI 补救，不能将“定向测试已交付”写成“本机代码收口完成”。 +没有完整打包三类正式镜像，没有做性能压测、多节点拓扑、断电或进程强杀恢复实验。 + +Java 命令在仓库根目录执行，临时路径同时传给 Maven 与 Surefire fork： + +```bash +mvn -o test -pl hugegraph-server/hugegraph-test -am -P unit-test,rocksdb \ + -Dtest=RocksDBSessionsTest,RocksDBSnapshotRestoreTest \ + -Dsurefire.failIfNoSpecifiedTests=false -DfailIfNoTests=false \ + '-DargLine=-Xmx512m -Djava.io.tmpdir=/tmp/topling-local-20260927/java-delivery-final' \ + -Djava.io.tmpdir=/tmp/topling-local-20260927/java-delivery-final \ + -Dmaven.javadoc.skip=true -ntp +``` + +三名独立只读审查者检查行为和数据安全 diff，并对修正后的受影响部分复审。 +审查发现的锁转移、direct marker 绕过、配置/存储 symlink、根目录、并发 marker、 +诊断大小写和同文件系统 bind 漏检均已处理；最后结论见交付记录。 + +Docker Server 的 ABI CI 探测绕过 entrypoint，新增根校验使未准备默认路径的旧 fixture 失败。 +补齐临时 probe 的 data/WAL 空目录后，真实 selector/marker/native probe 通过， +未关闭校验或改写用户图配置；日志 docker-ci-server-probe-final.log。 + +## CI 补救与完成状态更正 + +2026-09-27 核对 `cf25a438a50f93f96efce74b9e4b0cacce4fac91` 的全部 18 个失败 job， +实际只有两条根因，不是 18 个互不相关的故障： + +- 16 个 job 在 package/install 后编译新增 PD 测试失败。Spring Boot repackage 把主 artifact + 改成 BOOT-INF/classes 布局，compile 阶段的 target/classes 能解析，打包后的依赖 JAR 不能解析。 + 修复为薄主 JAR + exec 可执行 JAR；发行包只带 exec，启动脚本保持匹配唯一 Boot JAR。 +- 2 个 Mac job 在 `MultiGraphsTest.testCreateGraphsWithMultiDisksForRocksDB` 失败。 + 新恢复 guard 把既有 checked RocksDBException 契约改成 BackendException，破坏锁争用/shared CF 分支。 + 公共 open 现保留 checked IOError，并用结构化 contention 分类，拒绝目录名里的错误子串绕过。 + 只有活 native owner 可复用，检查、复制、session 初始化与缓存登记对同一 owner 的恢复互斥。 + +失败 run：[Server](https://github.com/hugegraph/hugegraph/actions/runs/36270918750)、 +[PD/Store](https://github.com/hugegraph/hugegraph/actions/runs/36270918555)、 +[Commons](https://github.com/hugegraph/hugegraph/actions/runs/36270918548)、 +[cluster](https://github.com/hugegraph/hugegraph/actions/runs/36270918538)、 +[CodeQL](https://github.com/hugegraph/hugegraph/actions/runs/36270918537)、 +[dependency-check](https://github.com/hugegraph/hugegraph/actions/runs/36270918547)。 + +| 补救验证 | 当前结果 | 证据文件（同一 /tmp 目录） | +| --- | --- | --- | +| clean PD package 最小复现 | 修复前缺两个符号，修复后 BUILD SUCCESS；未 clean 的旧 classes 曾掩盖失败 | repair-pd-clean-package-before.log / repair-pd-clean-package-after.log | +| 完整 root clean install | 所有模块成功；首次离线缺未缓存 shade plugin，联网补齐后实跑成功 | repair-root-clean-install-online.log | +| PD artifact / distribution | 主 JAR 有普通 classes，exec 有 BOOT-INF；发行包唯一 exec，manifest JarLauncher/HugePDServer 正确 | root install 后 zip、manifest 和 dist 核对 | +| PD 新增 mock 回归 | IndexAPIClusterStateTest 2/0/0/0 | repair-pd-index-tests.log | +| PD 实际发行包启动 | Mac JDK 11，独立临时端口/data，java -jar 后 actuator health UP；仅终止本次进程 | repair-pd-boot/server.log / result.json | +| 标准多盘最小复现 | 修复前 1 个失败；首次契约修复后通过，最终版本纳入完整核心套件 | repair-multidisks-before.log / repair-multidisks-after.log | +| 标准 CoreTestSuite | 最终源码 suite 818/0/0/42；合并执行命令整体失败因 session 单测用了旧共享 temp，不能声称该命令退出 0 | repair-core-and-unit-final.log 及 CoreTestSuite XML | +| 标准 session/helper | 最终版本用新专属 temp 43/0/0/0，命令 BUILD SUCCESS；合并命令的旧 pending/缺 checkpoint 失败日志保留，未删除旧 marker 绕过保护 | repair-unit-tests-complete.log / repair-core-and-unit-final.log | +| 真实 TP helper / adapter | 最终版本 27/0/0 与 1/0/0；首个仅锁复制的补丁被真实 TP 并发用例证伪，扩展至初始化后通过 | repair-tp-helper-final.log（失败）/ repair-tp-helper-complete.log / repair-tp-adapter-final.log | + +三名只读审查者复查本次最终 diff；发现的异常消息误判与检查/复制/初始化竞态已修复，最终均无阻塞发现。 +此次保证同一 owner 的初始化与恢复互斥；不同副本后续独立恢复的引用重绑定属于既有共享生命周期边界， +未扩展为支持任意副本并发恢复。真实服务验收应沿实际 Store 生命周期执行，不能泛化单个并发用例。 +第一轮本机补救验证完成，当时代码 head 为 `dfd4ce07e98e5846a2a093750f4b59a3061ab393`，PD 打包提交 `5401221996d7af71a0ea9a4243b5f0d1529237cc`。 +最终格式、完整 clean compile 与 git diff --check 通过。71 个相对链接/锚点与 Linux bash 命令语法通过， +四份历史档案保持逐字节一致；一名文档审查者指出的临时目录隔离问题已修正。 +当前整体收口仍待相关远端 CI，不能仅凭本机验证、推送或旧 goal complete 再标记完成。 +Linux 接收新源码后复跑多盘/恢复锁、PD 打包与实际服务场景；服务器结果仍只在 linux.md 维护。 + +## 第二轮 CI 补救：安全启动 fixture 与依赖清单 + +2026-09-27 再核对 `e4fef4b95c82b561d5abac021acccd974ab11a24`：38 项检查中 36 成功、2 失败。 +第一轮修复的 PD 编译和 Mac 多盘任务已通过；剩余失败另有根因，不能仍称全 CI 已收口。 + +- [Server RocksDB job](https://github.com/hugegraph/hugegraph/actions/runs/36295349099/job/108559275720) + 在 package 的 `BUILD SUCCESS` 后，被安全配置测试的 + `FAIL: valid Java security properties did not reach server startup` 中断。 + 本机同样复现；实际 stderr 为 `unreadable configuration: .../missing-rest.properties`。 + 新 provider selector 在 JVM 前读取 REST，旧 positive/disabled 用例却故意传缺失 REST。 + 修正 fixture：专属临时 REST + 存在的空 graphs 目录，继续传缺失 Gremlin YAML。 + 仍同时断言 YAML 加载失败、bootstrap main、Server main，未修改生产校验或安全规则； + 失败时打印捕获 stderr，避免 CI 只留泛化错误。 +- [dependency-check](https://github.com/hugegraph/hugegraph/actions/runs/36295348928/job/108553061001) + 同样在 Maven 成功后因清单 diff 退出 1，仅少 `slf4j-api-2.0.9.jar`。 + 本 PR 的 `0d2d334c5` 已把 minicluster 的 SLF4J 改为 1.7.25，遗漏同步 known 清单。 + 全 reactor 的 runtime copy-dependencies 复现唯一删除，无其他漂移；删除这一过期项。 + 与 PD thin/exec 无关，没有通过加入不需要的库消除 diff。 + +截图中的 assembly 父 POM 告警仍可观察：调试日志确认模型解析请求了字面 `${revision}` 的根 POM； +同次实际打包退出 0，Server 发行包/tar 成功生成。它们未触发上述两个 job 的退出， +此轮没有为消除告警改造共有 POM/assembly 布局，也不将打包成功推导为完整服务验收。 + +| 验证 | 结果 | 证据（/tmp/topling-local-20260927） | +| --- | --- | --- | +| 原安全 fixture | 同 CI 稳定失败，stderr 指向缺失 REST；诊断临时副本保留失败文件 | security-fixture-before.log / security-fixture-before-debug.log | +| Mac JDK11 完整安全配置脚本 | 退出 0，PASS Java security properties and startup wiring | security-fixture-mac-after.log | +| Linux amd64/JDK11 同脚本 | Mac OrbStack 模拟 amd64、2 CPU/2 GiB/禁网，真实发行包复制进独立容器；退出 0，完整脚本 PASS | security-fixture-linux-after.log | +| runtime 依赖对照 | 原 CI generator 成功，清单唯一删除；check_dependencies.sh 退出 0 | inventory-copy.log / inventory-before-sorted.diff / inventory-check.log | +| Bash 与 shellcheck | bash -n 通过，无新增 shellcheck 发现 | security-shellcheck-before.json / security-shellcheck-after.json | +| 独立只读审查 | 两文件 diff 无阻塞，保留全部安全断言、有效配置隔离和库版本归属 | security_fixture_review | + +本轮代码提交 `218f52309a7109d19d76b3161674d24c86c2c4c3`，生产行为未改。Mac/Linux 完整安全脚本、实际依赖对照、 +独立只读审查均通过；格式、完整 clean compile、bash -n 与 diff --check 通过， +73 个相对链接/锚点及四份历史档案一致性通过。 +第二轮当时的下一动作是核对提交及文档的推送结果、新 head CI；其结果与后续修正见下方第三轮。 + +## 第三轮 CI 补救：真实启动与 backend 初始化 + +`89979877b839ebcf415608e4eb601fa5bf3f7970` 的 CI 最终 37 成功、1 失败。 +[唯一失败 job](https://github.com/hugegraph/hugegraph/actions/runs/36301842353/job/108581358948) +已通过第二轮安全配置测试和清单门禁，随后真实启动套件 6 通过/7 失败。 +此前本机验证没有覆盖这条完整启动链，不能从安全脚本通过推导为真实 Server 启动通过。 + +根因已在隔离发行包稳定复现：安全启动的 provider admission 会先创建 data/WAL 根与 marker， +此时没有 backend CF；startup fixture 却凭 `rocksdb-data` 目录存在跳过 `init-store`。 +后续真实错误为 `The backend store of 'DEFAULT-hugegraph' has not been initialized`， +daemon、前台、HTTP 和 cron 失败由此连带触发。 + +修正仅在 `test-start-hugegraph.sh`：先 cleanup，再始终执行既有 `init-store.sh`； +实际 backend 是否初始化由原生表/CF 检查决定，不给目录/marker 赋初始化完成语义。 +初始化失败打印日志，每个失败 section 首次打印启动日志。 +信号用例要求真实 Server Java PID 尚存活，HTTP 未就绪计失败； +watchdog 超时单独判失败，不能把它杀死 wrapper 的非零退出误作传播成功。 +未改变生产 provider/root/security 校验，也未扩展 InitStore 的版本不匹配处理。 +后续 API、install 和 native smoke 脚本已核查均会初始化，没有其他同类目录跳过路径。 + +| 验证 | 结果与范围 | /tmp/topling-local-20260927 证据 | +| --- | --- | --- | +| 未初始化根的实际启动 | invalid security 启动先认领 roots，CURRENT 为 0；daemon 启动失败，日志为 backend 未初始化 | startup-before-container.log / startup-before-server-logs | +| 初始化后的对照 | 同样先认领 roots，再 init-store；实际 daemon 成功，/versions 返回 200 及版本 JSON | startup-after-container.log / startup-http-after.json | +| 完整 CI 启动顺序 | 实际完整安全脚本 PASS；确认 roots 已存在但无 CURRENT；完整 startup suite 16/0 | startup-chain-security.log / startup-suite-after.log | +| 已初始化 backend 重复执行 | 同一 backend 再执行完整 suite，16/0；真实 daemon/前台、HTTP、cron、SIGKILL 137、SIGTERM 143 全过 | startup-suite-repeat.log / startup-suite-repeat-server-logs | +| timeout 反例 | 自然退出 7 保留原码且非 timeout；挂住子进程被 watchdog 终止时 timeout=true，不当成功 | startup-wait-probe.log | +| 静态及独立审查 | bash -n、shellcheck 增量无新发现；只读复审修复 timeout 误判后无阻塞 | startup-shellcheck-final.json / startup_review | + +完整套件只在专属 Linux arm64/JDK11 容器执行，真实 crontab/fuser/curl/ps；2 CPU/2 GiB，运行时禁网。 +镜像 `local/hg-startup-check:20260927` manifest 为 +`77a468132b9e43192b6a8a3b4dae8e980934f31a90b320873acd62ee3e404541`。 +发行包只读挂载后复制到容器临时目录，进程、端口和 cron 清理只作用于该容器，未运行会清理全局资源的脚本于宿主。 +这是本机核心正确性证据，不能推导远端 Linux 或性能通过。 + +本轮修复提交 `8d92909b78f05e762e3e741a0949d6d33dac9374`，本机完整真实回归、独立复审、 +格式/完整 clean compile 与静态检查已通过。当前待核对提交推送和新 head 的远端 CI。 +不再次提前标记整体代码收口完成。 + +## 本机核心实测身份 + +Mac arm64，Homebrew Java 11.0.32.1、Maven 3.9.16。OrbStack 容器运行 Linux amd64/JDK 11， +CPU 架构经模拟，限 2 CPU/2 GiB/Java heap 512 MiB，测试时禁网；不用于性能结论。 +镜像 `local/topling-core-check:20260927` 只提供 JDK/native 依赖，源码和编译 classes 从工作树只读挂载。 +镜像 manifest `312e13b20d9e990dbd3b01bbdfb963b4464c7fe6f037b33a5009fa367ee94737`。 + +- JNI JAR：`rocksdbjni-8.10.2-20260725.141011-1.jar`，SHA-256 + `86eb1bd3d9f84ef0dddd3fe95c640a6f145ca2d5a26a5ba628f1f298a2031fae`。 +- Linux native：SHA-256 `c25ff6e676290db6db47df0954640aa609c391450ec90e1a8eec1f87e174dd38`。 +- 实际 `/proc/self/maps` 与 JAR CodeSource 已检查；ldd 无 missing 依赖，Easy Migrate 配置取本仓库。 +- 挂载实测只在该次临时容器授予 SYS_ADMIN 以创建 bind mount；未修改宿主挂载或 Docker 全局设置。 + +需要核心 1+3+3 或容器实测时允许继续；性能及高资源非核心矩阵留给 Linux。 +本轮六项根因可由上述核心执行路径覆盖，因此没有增加无关拓扑。 + +## 交付记录 + +- 配置与根隔离：`8054b6552e67b872e601d3d3a8cb6021bab4f1aa`。 +- WAL 与 adapter：`4bf7612e2acaef4d230b9b581a2afd31591332fe`。 +- CI 与诊断：`ceea35428a1f182cda294a93636da3dd9c2a3438`。 +- Docker ABI fixture 修正：`741c64a5c3858d7d3489d209acec0935b0a8af58`，为完整代码交付 head；生产源码与此前三批保持一致。 +- 三名只读审查者在最终受影响 diff 复审后均无阻塞发现；未替代实际测试。 +- 新 fixture 的 executable bit 已核对并直接在 Linux 运行，避免 CI 裸调用权限失败。 +- 四批代码已非强制推送，ls-remote 与 PR #179 head 均为 `741c64a5c3858d7d3489d209acec0935b0a8af58`。 +- 68 个相对链接/锚点通过，四份历史档案与原基线逐字节一致；Linux 文档内 probe 原文提取实跑 adapter 1/0/0。 +- 一名独立文档审查者核查迁移与最终交接；指出的旧状态和 CI 新旧 head 歧义已修正。 +- 文档与关联 issue 进展随本轮发布；恢复时核对包含本记录的提交及 GitHub 实时状态，不继承其他版本或服务器验收结果。 +Linux 待验项与可执行命令见 [linux.md](linux.md)，Mac 完成不关闭它们。 + +## 历史开发证据 + +文档拆分提交为 7c7519b79；四份历史档案与 4e1db45215 基线原文逐字节一致。 +初始化阶段 59 个链接/锚点及一名只读审查通过，不是代码验证。 +旧 master 同步、Raft 测试和通用指标 patch 见 [历史开发交接](https://github.com/hugegraph/hugegraph/blob/4720da91b5a3c99b426c2ac035533ebc84d4e4cb/.goal-task/toplingdb-linux-closure/development-handoff-history-20260926.md)， +它们没有被移入此次 TP 代码交付，也不作为本次功能通过证据。 diff --git a/.goal-task/toplingdb-linux-closure/pr-split-handoff.md b/.goal-task/toplingdb-linux-closure/pr-split-handoff.md new file mode 100644 index 0000000000..d7848b47c5 --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/pr-split-handoff.md @@ -0,0 +1,135 @@ +# ToplingDB PR 拆分与 Apache 上游交接 + +更新于 2026-10-03。本文是新会话的执行入口，不要求读取原聊天记录。 +目标是把 [hugegraph/hugegraph PR #179](https://github.com/hugegraph/hugegraph/pull/179) +拆成约四个容易审查、能够逐个安全合入的 PR。分支创建在 **hugegraph/hugegraph（org）**， +子 PR 默认先指向 **org 的 master** 做实测与 review；最终合入端为 **apache/hugegraph 的 master**。 + +## 用户已确定的边界 + +- 每个子 PR 完成构建、相关实测与 CI 后，**逐个交给用户 review；收到针对该 PR 的确认后才合入**。 + 仓库写权限不等于合入授权。不要启用 auto-merge，也不要直接推送 Apache master。 +- 分支保存在 org，优先独立基于 org master，并核对与 Apache master 的差异。 + 只有证实存在代码或行为依赖才使用 stacked PR；转向上游时重新确认 base、补丁范围和 CI。 + 不追求四个 PR 行数相等，不允许先合入已知不安全的中间版本再由后续 PR 修补。 +- 保留原 `toplingdb` 和 PR #179 作为整合与审计参考；不 force-push、重排其历史或提前关闭它。 + 子 PR 使用新分支，发布前检查是否已有其他会话创建的拆分 PR，避免重复工作。 +- GitHub 和版本库操作全部使用 `gh`；不要直接调用 Git，也不要通过别名隐藏 Git 命令。 + `gh repo clone`、`gh pr checkout` 及 `gh api` 的内置能力可以使用。 +- 性能 #252 可选；正式 JNI 发布链 #213 独立跟进，不扩大为本轮必须完成的发行流水线。 + 但 Apache 子 PR 中的依赖来源、许可证标注和实际获取方式仍须清楚，不能用旧验收免除其审查。 +- 原 Linux goal 已按用户要求暂停。新会话开展拆分任务，不自动恢复旧 goal。 + 用户已取消 7% 周额度保留限制，允许用完剩余额度；不得擅自兑换额度重置。 + +## 固定参考与实时状态 + +| 对象 | 已核对的参考 | +| --- | --- | +| 源 PR | hugegraph/hugegraph #179，head 分支 `toplingdb`。源 PR 和初始子 PR 均指向 org master；子 PR 最终再向 Apache master 提交。 | +| 本次分析源码/文档快照 | `1d97657194a908e6701779064f6b8e101570e10d`；本交接发布会再增加文档提交。新会话必须重新读取源 PR head 并固定工作 SHA。 | +| 最后生产代码修复 | `3aa44152e8e13749d82ba58449b93254384ed0ed`，后续到上述快照仅文档变化。 | +| Apache master | 本次核对为 `176fb56dd747ef0f60a126cf721aa12d17627c31`，新会话重新核对，不能假定没有前进。 | +| 源 PR CI | **1d976571 的 38 项检查已全部成功**。旧“33 项排队”记录已经过时；新 head/子 PR 的 CI 必须重新查询。 | +| 历史 Linux 冻结基线 | `9d797c7608e244f03436ce11294d9bd72aba4d2d`；不是新子 PR 的开发 base。 | +| 临时 auth 分支 | `toplingdb-auth-cascade-20261003` 指向已同步进原分支的修复，是临时隔离分支，不是四个上游子 PR 之一。 | + +旧 API 失败已经归因并修复：master 的项目删除重建正常，源 PR 的 FINISHED 清理使 schema +缓存变冷，隐藏关联边未被枚举，留下 access/belong。保留清理并修正级联删除后通过。 +**必须将请求清理、必要的 schema/index 修正和 auth 配套回归放在同一完整单元。** +曾用 `primitive()` 排除负 ID 范围的候选 `43b88c1` 不完整：真实用户标签为 -27，仍会漏删 +belong。最终实现仅排除特殊 OLAP 标签；不要恢复旧候选或仅摘取不完整的中间提交。 + +## 建议四个功能单元 + +以下是待验证的逻辑划分，本会话尚未构造四个子分支，也没有任何“每层 CI 已通过”的结论。 +路径均相对仓库根；同一个文件可能必须按修改块拆分。 + +| 单元 | 范围与源码入口 | 独立合入条件 | +| --- | --- | --- | +| 1. 标准运行时依赖 | 根/PD/Server/Store POM、Store `RocksDBMetricsConst`、依赖清单及 LICENSE/NOTICE。PD RocksDB 6.29.5、Store 7.7.3 → 8.10.2，Server 原已为 8.10.2。 | 默认 RocksDB 构建、服务及升级兼容正常；新建库重启不能替代旧数据打开验证。JRaft 1.3.11/1.3.13 → 1.3.14 是否必要须另证，非必要不夹带。 | +| 2. 事务与关闭生命周期 | Server `HugeFactory`、`StandardHugeGraph`、`BackendSessionPool`、REST `ApplicationConfig`、auth `ContextTask`、必要的 GraphTransaction/GraphIndexTransaction 修正；Store grpc/扫描与关停修复。 | 标准模式独立通过 Core/API、关闭及并发回归；项目和用户关联删除均正确，其他对象保留。不要把 auth 修复与触发它的清理接口拆开。 | +| 3. Snapshot/WAL 安全 | Server rocksdb 模块的 `RocksDBSnapshotRestore`、`OpenedRocksDB`、`RocksDBStdSessions`，`RocksDBStore` 的恢复/锁相关修改。 | 标准 provider 可独立验证成功与故障恢复、独立/嵌套 WAL、材料保留、锁/并发；不承诺尚未实现的全图原子恢复。 | +| 4. 完整 TP 接入 | provider 选项、TP truncate 差异、JNI 选择、ABI 与所有数据/WAL 根预检、三组件包、启动脚本、Docker、对应 CI 和产品文档。 | 合入时 TP 是完整可选能力，默认标准包不混入 TP；真实 JNI、拒绝路径、服务/重启/关闭通过。 | + +预期在核对两边 master 后，独立准备单元 1～3，先以 org master 为子 PR 的默认 base。 +单元 4 依赖必要前置；org 已合入的前置若尚未进入 Apache，上游仍须显式处理这条依赖。 +发现实际依赖时再增加 stack 边，并在 PR 描述中写出 base、前置 PR、合入顺序和重新验证范围。 +不要仅为“平行”复制公共实现，也不要仅为“stack”制造不必要串行等待。 + +特别注意以下跨文件依赖： + +- `RocksDBStore` 同时含 provider/truncate、opened/session 和恢复锁修改，不能整文件随意归入一层。 +- `RocksDBSessionsTest` 同时含原有测试、新 WAL 用例和 TP adapter 分支测试，需要按测试目的分配。 +- `OpenedRocksDB` 的关闭和恢复锁移交是配套实现，锁释放顺序不能拆散。 +- 当前标准 runtime CI 也调用 `preload-topling.sh`；workflow 与它依赖的脚本/配置必须在同一层可用。 +- `testAdapterToplingTruncateWithMultipleKeys` 在标准 JNI 下也能检查 Java 分支；名字不证明已加载 TP。 + 真正 TP 验证另核唯一 JNI JAR、实际 native maps 和哈希。 + +## 规模与保留范围 + +在 1d976571 相对 176fb56dd 的快照中：209 文件，+16,167/-646 行，228 个提交。 +生产 Java 34 文件 +1,329/-311；运行脚本 18 文件 +1,466/-49；测试 62 文件 +6,016/-181； +构建/CI/配置 41 文件 +1,881/-97；文档/许可证文本 42 文件 +5,475/-8；另有 11 张图片和 1 个 JNI JAR。 +测试包括 44 个 Java 文件和 18 个 Shell 文件，不全是 TP 专属。 + +四单元新增量曾粗估约 30、2,300、1,600、8,500 行；不是配额或最终补丁大小。 +第一个还需补升级验证，最后一个含大量启动/发行测试与文档。不要为了接近估算而遗漏实现。 +`.goal-task/`、`.specs/` 和 `docs/toplingdb/images/` 合计约 3,666 行历史/规划/绘图材料， +应保留在审计来源中，默认不搬入 Apache 子 PR。产品操作说明、必要许可证和有效测试必须随实现。 +实验 JNI JAR 约 9.1 MiB，图片约 12.6 MiB，均未计入文本行数；交付方式单独给用户 review。 + +## 已有验证与不能外推的结论 + +实测结论、运行产物身份与原始证据位置只维护在 [linux.md](linux.md)，Mac 对照见 [mac.md](mac.md)。 +源码重组或变更 base 后，按实际影响复测；源 PR 绿灯不能证明拆分后的中间树安全。 +已有新库结果不证明 PD/Store 旧数据升级或降级兼容，这是单元 1 需要补齐的边界。 + +新环境先核架构、ABI、CPU 与权限，不能将 Linux x86_64 native 身份套到其他架构。 +旧 evidence 绝对路径和 /tmp 脚本可能不可用；用仓库测试、CI 与 linux.md 重建必要验证。 +未取得原始材料时标明“旧记录”，不宣称独立复核。首次失败保留，大日志/数据库/镜像不提交。 + +## 新会话启动与执行顺序 + +1. 默认只读根 `AGENTS.md`、state.md 和本文；linux/mac/todo/lessons 按待拆分范围取用， + 再读相应模块 AGENTS 和 CONTRIBUTING。不递归加载任务目录；历史材料仅在具体取证时从固定提交读取。 + 检查已有本地改动/分支/资源及 org 和上游现有 PR，不覆盖其他会话的工作。 +2. 用 gh 刷新源 PR、org master、Apache master 和检查结果，记录不可变 SHA。需要新源码时使用完整 checkout； + GitHub archive 的 export-ignore 曾漏掉 install-dist 文件，不能直接拿不完整归档构建。 +3. 生成逐文件/修改块归属清单：子 PR、理由、依赖、配套测试/文档、未纳入及原因。 + 先验证 1～3 可否独立；明确真正的 stack 关系，不按旧提交顺序机械 cherry-pick。 +4. 在 org 创建新分支，构造第一个完整子 PR 并干净验证，再逐步推进。 + 默认先向 org master 创建 draft PR 用于实测与 review；具备提交条件后再向 Apache master 提交。 + 上游 PR 可复用 org 分支，但必须重新检查上游差异与 CI，不能把 org 的绿灯直接当成上游通过。 + 每个子 PR 标明源码身份、实际执行/skip 计数、标准/TP 覆盖边界、风险及与后续 PR 的关系。 +5. 生产行为或持久化修改由三名独立只读审查者检查，修改后复审与实测。重任务/测试服务串行，启动清理只在专属容器。 + 发布前按仓库要求运行格式、干净编译及相关测试；Commons 需显式启用测试。不要为绿灯删除断言或放宽例外。 +6. 每个子 PR 满足检查后，向用户提供仓库、PR、head 和验证结果，等待该 PR 的合入确认。 + org 与 Apache 是不同的合入操作，不能将一边的确认视为另一边已获授权。 + 确认后使用正常合入流程，记录 merge SHA；再刷新依赖 PR 的 base、差异和 CI。禁止绕过必需检查/使用管理权限强合。 +7. 全部所需单元合入后，核对最终 Apache master 的功能集合与源 PR 意图，执行标准/真实 TP 整合验证。 + 原 PR #179 和未完成 issue 的关闭/清理再向用户确认，保留审计索引。 + +首次核对命令示例（在适合的新目录运行 clone）： + +```bash +gh pr view 179 --repo hugegraph/hugegraph --json headRefOid,baseRefOid,statusCheckRollup +gh api repos/hugegraph/hugegraph/commits/master --jq .sha +gh api repos/apache/hugegraph/commits/master --jq .sha +gh api --paginate 'repos/hugegraph/hugegraph/pulls/179/files?per_page=100' +gh repo clone hugegraph/hugegraph hugegraph-source -- --branch toplingdb +``` + +新环境先检查 gh 版本和认证。org 内的初始 PR 可使用 +`gh pr create --repo hugegraph/hugegraph --base master --head --draft`。 +最终提交上游时，当前 gh 的 `pr create --head :` 帮助提示不支持组织 owner； +可用 `gh api` 创建 cross-fork draft PR，目标为 `apache/hugegraph`、base 为 master、 +head 为 `hugegraph:`，并核对返回的 base/head repo。 +需要保存提交时可使用 gh Git Data API 创建 tree/commit/ref；更新 ref 必须核对预期父提交并 `force:false`。 +不要把新环境的工具限制自动等同没有仓库写权限，更不要绕过其执行策略。 + +## 需要提出具体方案后交用户判断的事项 + +- 若四单元无法各自独立构建/合入，说明真实依赖以及增加 stack 或调整数量的代价，再决定拆分边界。 +- 若旧数据升级验证失败、需不可逆迁移、改变默认行为或删除已有支持，先提供复现和迁移/回滚方案。 +- JNI 获取与交付方式、必要的上游网站文档配套 PR，以及不能从来源解释的许可证标注，均应在相关子 PR review 中明确。 +- org 分支及初始 PR、Apache 最终合入端、逐 PR 确认已经由用户决定，不重复询问。 diff --git a/.goal-task/toplingdb-linux-closure/state.md b/.goal-task/toplingdb-linux-closure/state.md new file mode 100644 index 0000000000..8068385250 --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/state.md @@ -0,0 +1,34 @@ +# ToplingDB 当前入口 + +更新于 2026-10-03。原 Linux goal 按用户要求暂停；下一任务是拆分 PR #179。 + +## 默认阅读范围 + +新会话先读仓库 AGENTS.md、本页和 [拆分交接](pr-split-handoff.md)。 +不要递归加载本目录、两套环境记录或历史文件。其余材料仅在处理对应问题时读取。 + +| 文件 | 唯一职责 | 何时读取 | +| --- | --- | --- | +| [pr-split-handoff.md](pr-split-handoff.md) | 拆分边界、依赖、验证及合入顺序 | 拆分任务启动时 | +| [todo.md](todo.md) | 未完成问题的归属和 issue 索引 | 判定阻塞或安排后续时 | +| [linux.md](linux.md) | Linux 源码/产物身份、实际结果与证据位置 | Linux 复测或核查结论时 | +| [mac.md](mac.md) | Mac 开发、容器测试及其适用边界 | 在 Mac 执行或查对应证据时 | +| [lessons.md](lessons.md) | 已验证的操作陷阱，含平台分节 | 执行相关操作前按节读取 | + +## 当前事实与决定 + +- 源分支 `toplingdb`、[PR #179](https://github.com/hugegraph/hugegraph/pull/179) 保留，不重写历史。 +- 最后生产代码为 `3aa44152e8e13749d82ba58449b93254384ed0ed`；修复已发布，标准/真实 TP 实测通过。 + 源快照 `1d976571` 的 38 项 CI 已全部成功；后续 head 和拆分后的 CI 必须重新查询。 +- 分支建在 org，子 PR 默认先向 `hugegraph/hugegraph master` 提交并实测，最终合入 Apache master。 + 每个 PR 交用户 review，收到对应确认后才合入；具体流程见拆分交接。 +- TP 适配及本 PR 新增共用代码的回归仍负责；此前已有通用问题独立跟进。 + 性能 #252 可选，正式 JNI 发布链 #213 独立后置，均不阻塞本轮 Linux 任务。 +- 用户允许用完剩余额度；版本库操作使用 gh。重任务串行，启动清理只在专属容器执行。 + +## 历史取证 + +当前目录只维护上述六份文件。历史快照、旧入口和旧证据索引保留在 +[精简前固定提交](https://github.com/hugegraph/hugegraph/tree/4720da91b5a3c99b426c2ac035533ebc84d4e4cb/.goal-task/toplingdb-linux-closure)。 +仅调查具体旧结论时按文件读取；旧授权、资源状态和失败状态均不作为当前计划。 +原始日志、数据库、镜像仍留在原专属 evidence/data 目录，本次整理不删除它们。 diff --git a/.goal-task/toplingdb-linux-closure/todo.md b/.goal-task/toplingdb-linux-closure/todo.md new file mode 100644 index 0000000000..32518baea1 --- /dev/null +++ b/.goal-task/toplingdb-linux-closure/todo.md @@ -0,0 +1,33 @@ +# ToplingDB issue 归属与依赖索引 + +新会话拆分计划见 [pr-split-handoff.md](pr-split-handoff.md)。当前范围见 [state.md](state.md)，实测及下一动作只在 [linux.md](linux.md) 维护。 +本表把 TP 新增/放大的问题与通用独立调查分开；发现新归因时按源码证据重新分类。 + +| Issue / 范围 | 归属 | 依赖与记录位置 | +| --- | --- | --- | +| #250/#251/#253 配置、目录、挂载、provider | TP 适配门禁。 | [Linux 清单](linux.md#验收清单)，源码变化时回归真实拒绝与数据不变。 | +| #254 truncate / #255 diagnostic | TP JNI 与 adapter 门禁。 | [Linux 清单](linux.md#验收清单)，需要真实身份；合成 CF 精确例外不覆盖其他错误。 | +| #249 WAL 扩展 457295ac8 | PR 新增共用代码，仍负责其回归。 | [Linux 故障证据](linux.md#真实服务与故障证据)；发布/冷启及 pending 并发已完成标准/真实 TP 对照，待证据与交付复核。 | +| #213 JNI 来源、许可、支持下限 | 正式发行独立后置，不阻塞当前任务。 | [Linux 发行核查](linux.md#jni-来源与正式发布)，由 producer 提供材料。 | +| #252 三轮标准/TP 对照 | 可选后置，不阻塞当前任务。 | 决定执行时另按固定 workload/配置/资源记录，不依赖通用在线恢复改造。 | +| #212 真实 native 生命周期 | 新的 TP 独有错误仍归适配；通用请求排空独立。 | [Linux 生命周期记录](linux.md#独立调查与隔离候选)，保留冻结首次失败。 | +| #248 clear 首次重启不可见 | 通用调查，旧失败保留。 | [Linux 对照记录](linux.md#独立调查与隔离候选)，不同源码/schema/部署不能覆盖旧失败。 | +| 历史 ProjectApiTest CI 回归（已修复） | 已由 master/单点实测确认本 PR 的 FINISHED 清理触发，属于当前任务。 | [CI 归因与计划](linux.md#2026-10-03-ci-归因与后续计划)，用户 -27 缺口已由红绿实测闭合；3aa44152 已发布到 PR #179，三人复审、标准55/TP5及最终双 provider 服务通过；源快照 1d976571 的 38 项 CI 已通过；拆分后需按新源码重验。 | +| master 同步 | 保留双方改动及四项修复的发布兼容性。 | [Linux 源码状态](linux.md#当前范围与源码)，不替代冻结 SHA 实测。 | + +## 通用问题独立跟进 + +| 问题 | 现有归属与边界 | +| --- | --- | +| snapshot 缓存/全图协调、保存对象生命周期 | 相关机制在标准源码早已缺失；旧 epoch 拒绝断言仅在未提交候选实测，详见 [源码归因](linux.md#问题归属依据)。 | +| 跨 Server schema cache | [Apache #3235](https://github.com/apache/hugegraph/issues/3235)、[PR #3237](https://github.com/apache/hugegraph/pull/3237)、[组织 PR #236](https://github.com/hugegraph/hugegraph/pull/236)。 | +| clear 缓存、truncate 吞异常、同名图重建 | 分别沿组织 #242/#243/#244，schema PR 不覆盖全部行为。 | +| Store session 指标 / JVM 不退出 | 组织 #241/#211；标准也复现，旧补丁不夹带。 | +| 健康检查与 Gremlin 停机排空 | 开库拒绝后 versions 200、在途请求 500 属通用语义；详见 [独立调查](linux.md#独立调查与隔离候选)。 | +| Store 地址、路由/扫描、Loader | 组织 #245；Apache #3124/#3130，历史 channel refresh 保持独立。 | +| HStore 图快照与完整 HA/部署配置 | 组织 #246/#247；PR #235 单机备份不能当 HStore 图协议，单宿主不证明物理多机 HA。 | +| 临时端口 / HStore 联合索引 | 组织 #216/#217，按原 issue 跟进。 | + +父汇总为 [#240](https://github.com/hugegraph/hugegraph/issues/240)，#214 仅作历史里程碑。 +旧完整索引保留在 [同期归档](https://github.com/hugegraph/hugegraph/blob/4720da91b5a3c99b426c2ac035533ebc84d4e4cb/.goal-task/toplingdb-linux-closure/state-history-20261002.md#todomd-同期原始索引) 与 +[todo-history-20260926.md](https://github.com/hugegraph/hugegraph/blob/4720da91b5a3c99b426c2ac035533ebc84d4e4cb/.goal-task/toplingdb-linux-closure/todo-history-20260926.md)；不将旧授权、状态或门禁作为当前安排。 diff --git a/.specs/hugegraph-server/ToplingDB/design.md b/.specs/hugegraph-server/ToplingDB/design.md new file mode 100644 index 0000000000..38685cbbcd --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/design.md @@ -0,0 +1,159 @@ +# Design of ToplingDB Easy Migrate + +## Overview + +ToplingDB is an optional RocksDB-compatible runtime for HugeGraph. HugeGraph +Server, PD, and Store keep their existing RocksDB Java API calls. The +integration does not add a Java provider SPI and does not create or manage a +`SidePluginRepo`. + +Easy Migrate is activated before the JVM starts: + +1. The installation layer prepares the selected JAR, native library, and Web + resources. +2. The startup layer selects `rocksdb` or `topling` from the component + configuration. +3. For `topling`, the startup layer exports + `TOPLINGDB_EASY_MIGRATE_CONF`, `LD_LIBRARY_PATH`, and `LD_PRELOAD`. +4. The JVM opens and closes RocksDB through the existing Java API. +5. The Topling native hook imports the YAML options and tracks the normal + DB/column-family lifecycle. + +## Goals + +- Keep the standard RocksDB Java API and the existing HugeGraph storage + lifecycle. +- Make ToplingDB an explicit, startup-time selection. +- Apply component-specific YAML configuration through Easy Migrate. +- Preserve a strict standard RocksDB fallback when ToplingDB is not selected. +- Keep Server, PD, and Store shutdown behavior uniform. + +## Runtime Architecture + +```mermaid +sequenceDiagram + autonumber + participant User + participant Install as prepare-topling.sh + participant Start as component start script + participant Preload as preload-topling.sh + participant JVM as HugeGraph JVM + participant Hook as Topling native hook + participant Rocks as RocksDB Java API + + User->>Install: Prepare selected runtime + User->>Start: Start Server, PD, or Store + Start->>Preload: Select configured provider + alt provider is rocksdb or unset + Preload->>Preload: Keep standard RocksDB runtime + else provider is topling + Preload->>Preload: Validate prepared Topling runtime + Preload->>Preload: Export TOPLINGDB_EASY_MIGRATE_CONF + Preload->>Hook: Preload native library + end + Start->>JVM: Launch component + JVM->>Rocks: Normal RocksDB open/CF operations + Hook->>Hook: Apply YAML options and retain DB/CF state + JVM->>Rocks: Normal CF/DB close + Hook->>Hook: MaybeForgetCF / MaybeForgetDB +``` + +HugeGraph Java code must not: + +- define or load `RocksDBProviderLoader` or `ToplingRocksDBProvider`; +- reflectively construct a `SidePluginRepo`; +- call `SidePluginRepo.openDB()` or `closeAllDB()`; +- treat ToplingDB as a separate service process. + +## Component Boundaries + +| Component | RocksDB role | Easy Migrate configuration | +|---|---|---| +| Standalone Server | Opens its local RocksDB backend | `conf/toplingdb.yaml` | +| PD | Opens PD metadata RocksDB | `conf/rocksdb_pd.yaml` | +| Store | Opens Store data RocksDB | `conf/rocksdb_store.yaml` | +| HStore Server | Remote client; no local RocksDB | Not applicable | + +HStore does not make HugeGraph Server a local RocksDB owner. ToplingDB for a +distributed deployment applies to PD and Store, where RocksDB is actually +opened. + +## Configuration Contract + +`rocksdb.provider` is the explicit runtime selector: + +```properties +# Default +#rocksdb.provider=rocksdb + +# Optional ToplingDB runtime +rocksdb.provider=topling +``` + +When `topling` is selected, the startup script must export a readable, +component-specific YAML path: + +```bash +export TOPLINGDB_EASY_MIGRATE_CONF=/path/to/conf/toplingdb.yaml +``` + +The YAML keeps `DBOptions.default` as the global fallback for databases that do +not have a more specific mapping. `DBOptions.log` is a deliberate specialized +profile and must remain distinct. + +## Open and Close Lifecycle + +The Java lifecycle stays unchanged: + +```text +RocksDB.open(...) / openReadOnly(...) + -> create, open, drop, and close column families + -> ColumnFamilyHandle.close() + -> RocksDB.close() +``` + +With Easy Migrate enabled, the native hook observes this lifecycle, applies the +matching options, retains native DB/CF state while it is in use, and runs +`MaybeForgetCF` and `MaybeForgetDB` during normal close. Java must not add a +second ownership path. + +The supported shutdown sequence is: + +```text +SIGTERM + -> HugeGraph/PD/Store graceful shutdown + -> normal CF and DB close + -> native MaybeForgetCF and MaybeForgetDB + -> JVM exit +``` + +Store already has a bounded shutdown wait. A timeout means the ordinary Store +shutdown did not finish within that boundary; it is not a reason to call +Topling-specific cleanup APIs. General thread-exit fixes are outside this +integration. + +## Data Compatibility and Rollback + +Runtime selection is not a data migration or rollback. The provided ToplingDB +configuration may enable Topling-specific WAL, SST, memtable, or compression +behavior. After ToplingDB writes data, changing only the JAR or provider is not +a safe rollback. + +Before migration, create a complete RocksDB-consistent checkpoint or backup. +To roll back, stop all writers, restore the complete pre-migration snapshot +into an empty data directory, restore the matching standard RocksDB runtime, +and validate the database before accepting traffic. + +## Verification + +ToplingDB DB/CF functional tests must run with +`TOPLINGDB_EASY_MIGRATE_CONF` set to a readable component configuration. +Unsetting the variable is allowed only for standard RocksDB tests or an ABI +diagnostic that does not open a database. + +The focused runtime test verifies: + +- the selected JAR and native library mapping; +- Easy Migrate configuration presence for ToplingDB; +- DB create, write, close, and reopen; +- CF create, drop, recreate, close, and reopen. diff --git a/.specs/hugegraph-server/ToplingDB/requirements.md b/.specs/hugegraph-server/ToplingDB/requirements.md new file mode 100644 index 0000000000..3c43a6c333 --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/requirements.md @@ -0,0 +1,48 @@ +# Requirements of ToplingDB Easy Migrate + +## Runtime Compatibility + +HugeGraph must support standard RocksDB by default and an explicitly selected +ToplingDB runtime without changing the RocksDB Java API used by Server, PD, or +Store. + +Acceptance criteria: + +- ToplingDB is selected only by component configuration. +- Missing or unreadable Easy Migrate configuration fails ToplingDB startup. +- Standard RocksDB does not require an Easy Migrate configuration. +- No HugeGraph Java code creates or manages a `SidePluginRepo`. + +## External Configuration + +The startup layer must set `TOPLINGDB_EASY_MIGRATE_CONF` to the configuration +for the component that owns RocksDB. The native hook must apply the YAML +options to normal RocksDB open and column-family operations. + +`DBOptions.default` remains the fallback for unmatched databases. +`DBOptions.log` remains a separate specialized mapping. + +## Component Ownership + +Standalone Server, PD, and Store may own local RocksDB instances. A Server +using the HStore backend is a remote client and must not be described as +opening a local RocksDB instance. + +## Graceful Shutdown + +SIGTERM must enter the existing HugeGraph, PD, or Store graceful shutdown, +which closes column families and databases through the normal RocksDB API. +Easy Migrate then releases native tracking through `MaybeForgetCF` and +`MaybeForgetDB`. + +No operation or test may require `SidePluginRepo.closeAllDB()`. The existing +bounded Store shutdown timeout remains a general lifecycle boundary, not a +ToplingDB-specific cleanup mechanism. + +## Verification + +- Standard RocksDB DB/CF tests run without Easy Migrate. +- ToplingDB ABI diagnostics may inspect only JAR/native availability and + mappings. +- Every ToplingDB DB/CF functional test runs with a readable + `TOPLINGDB_EASY_MIGRATE_CONF`. diff --git a/.specs/hugegraph-server/ToplingDB/task.md b/.specs/hugegraph-server/ToplingDB/task.md new file mode 100644 index 0000000000..757d4f88e1 --- /dev/null +++ b/.specs/hugegraph-server/ToplingDB/task.md @@ -0,0 +1,32 @@ +# Tasks of ToplingDB Easy Migrate + +## Runtime Integration + +- [x] Keep Server, PD, and Store on the existing RocksDB Java API. +- [x] Prepare the optional ToplingDB JAR, native library, and Web resources + before service startup. +- [x] Select ToplingDB explicitly from component configuration. +- [x] Export a readable component-specific + `TOPLINGDB_EASY_MIGRATE_CONF` before the JVM starts. +- [x] Let the native Easy Migrate hook apply options and track DB/CF lifecycle. +- [x] Keep `DBOptions.default` as the global fallback and retain the dedicated + `DBOptions.log` profile. + +## Component Integration + +- [x] Use `conf/toplingdb.yaml` for standalone Server. +- [x] Use `conf/rocksdb_pd.yaml` for PD. +- [x] Use `conf/rocksdb_store.yaml` for Store. +- [x] Keep HStore-backed Server as a remote client without local RocksDB + ownership. + +## Lifecycle and Testing + +- [x] Use normal RocksDB Java close paths during graceful shutdown. +- [x] Rely on native `MaybeForgetCF` and `MaybeForgetDB`; do not add Java-side + repository cleanup. +- [x] Verify JAR/native mapping without constructing a plugin repository. +- [x] Run every ToplingDB DB/CF functional test with Easy Migrate configuration + enabled. +- [ ] Keep general Store shutdown timeout and unrelated thread-exit fixes in + their existing workstream. diff --git a/README.md b/README.md index 0e48ced340..93e9aab17e 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,10 @@ --- -**Quick Navigation:** [Architecture](#architecture) • [Quick Start](#quick-start) • [Module Map](#module-map) • [Ecosystem](#ecosystem) • [For Contributors](#for-contributors) • [Community](#community) +**Quick Navigation:** [Architecture](#architecture) • [Quick Start](#quick-start) • +[ToplingDB](docs/toplingdb/README.md) • [Module Map](#module-map) • +[Ecosystem](#ecosystem) • [For Contributors](#for-contributors) • +[Community](#community) --- @@ -31,7 +34,12 @@ achieved through the powerful [Gremlin](https://tinkerpop.apache.org/gremlin.htm - **Schema Metadata Management**: VertexLabel, EdgeLabel, PropertyKey, and IndexLabel - **Multi-type Indexes**: Exact query, range query, and complex conditions combination query -- **Plug-in Backend Store Framework**: RocksDB powers standalone deployments and HStore powers distributed clusters. See the [backend evolution guide](hugegraph-server/README.md#backend-evolution-and-compatibility) for compatibility details. +- **Plug-in Backend Store Framework**: RocksDB powers standalone deployments + and HStore powers distributed clusters. ToplingDB is an optional + RocksDB-compatible runtime; see the + [ToplingDB documentation](docs/toplingdb/README.md). See the + [backend evolution guide](hugegraph-server/README.md#backend-evolution-and-compatibility) + for compatibility details. - **Big Data Integration**: Seamless integration with `Flink`/`Spark`/`HDFS` - **Complete Graph Ecosystem**: In/out-memory Graph Computing + Graph Visualization & Tools + Graph Learning & AI - **Dual Query Language Support**: [Gremlin](https://tinkerpop.apache.org/gremlin.html) (via [Apache TinkerPop 3](https://tinkerpop.apache.org/)) and [Cypher](https://en.wikipedia.org/wiki/Cypher_(query_language)) (OpenCypher) diff --git a/docker/README.md b/docker/README.md index a9d43dcb1f..c00051a244 100644 --- a/docker/README.md +++ b/docker/README.md @@ -18,6 +18,15 @@ cd docker Standalone uses `hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest}`. The HStore topologies use the matching `hugegraph/pd`, `hugegraph/store`, and `hugegraph/server` tags. Hubble is selected independently with `${HUBBLE_IMAGE:-hugegraph/hubble:latest}`. +ToplingDB is selected by injecting the documented image, provider, data-root, +marker, volume, and pull-policy variables in the Developers section. Its +published images target Linux x86_64 (`linux/amd64`); on macOS, use Docker +Desktop or OrbStack container mode and set `--platform linux/amd64` when the +engine does not select it automatically. Native macOS execution, including +Intel, is outside this ToplingDB matrix. Standalone selects Topling for the +Server, while HStore selects it only for PD and Store. The HStore Server +remains the standard image and never loads a local ToplingDB runtime. + ### Create the authentication environment Create `.env` once. Replace `replace-with-your-password` with an administrator password that you choose; the command generates and persists a random 32-byte JWT secret. For this simple single-quoted format, do not use a password that contains a single quote or newline. @@ -206,6 +215,147 @@ HUGEGRAPH_VERSION=1.7.0 \ docker compose -f docker-compose-hstore.yml up -d ``` +--- + +## Environment Variable Reference + +Configuration is injected via environment variables. The old `docker/configs/application-pd*.yml` and `docker/configs/application-store*.yml` files are no longer used. + +The generic Compose files pass explicit `rocksdb` defaults to each local +RocksDB owner. Selecting a `:topling` image therefore requires setting its +provider, data-root, marker, and provider-specific volume together; changing +only the image tag is intentionally not a provider switch. + +### PD Environment Variables + +| Variable | Required | Default | Maps To (`application.yml`) | Description | +|----------|----------|---------|-----------------------------|-------------| +| `HG_PD_GRPC_HOST` | Yes | — | `grpc.host` | This node's hostname/IP for gRPC | +| `HG_PD_RAFT_ADDRESS` | Yes | — | `raft.address` | This node's Raft address (e.g. `pd0:8610`) | +| `HG_PD_RAFT_PEERS_LIST` | Yes | — | `raft.peers-list` | All PD peers (e.g. `pd0:8610,pd1:8610,pd2:8610`) | +| `HG_PD_INITIAL_STORE_LIST` | Yes | — | `pd.initial-store-list` | Expected stores (e.g. `store0:8500,store1:8500,store2:8500`) | +| `HG_PD_GRPC_PORT` | No | `8686` | `grpc.port` | gRPC server port | +| `HG_PD_REST_PORT` | No | `8620` | `server.port` | REST API port | +| `HG_PD_ROCKSDB_PROVIDER` | No | `rocksdb` | `rocksdb.provider` | `rocksdb` or `topling`; set `topling` explicitly with a Topling image | +| `HG_PD_DATA_PATH` | No | Provider-specific path | `pd.data-path` | `/hugegraph-pd/pd_data` for RocksDB or `/hugegraph-pd/topling-pd-data` for ToplingDB | +| `HG_PD_ENFORCE_PROVIDER_MARKER` | No | `false` | Startup safety gate | Rejects an unmarked non-empty data path when `true`; enabled by Topling images | +| `HG_PD_INITIAL_STORE_COUNT` | No | `1` | `pd.initial-store-count` | Min stores for cluster availability | + +**Deprecated aliases** (still work but log a warning): + +| Deprecated | Use Instead | +|------------|-------------| +| `GRPC_HOST` | `HG_PD_GRPC_HOST` | +| `RAFT_ADDRESS` | `HG_PD_RAFT_ADDRESS` | +| `RAFT_PEERS` | `HG_PD_RAFT_PEERS_LIST` | +| `PD_INITIAL_STORE_LIST` | `HG_PD_INITIAL_STORE_LIST` | + +### Store Environment Variables + +| Variable | Required | Default | Maps To (`application.yml`) | Description | +|----------|----------|---------|-----------------------------|-------------| +| `HG_STORE_PD_ADDRESS` | Yes | — | `pdserver.address` | PD gRPC addresses (e.g. `pd0:8686,pd1:8686,pd2:8686`) | +| `HG_STORE_GRPC_HOST` | Yes | — | `grpc.host` | This node's hostname (e.g. `store0`) | +| `HG_STORE_RAFT_ADDRESS` | Yes | — | `raft.address` | This node's Raft address (e.g. `store0:8510`) | +| `HG_STORE_GRPC_PORT` | No | `8500` | `grpc.port` | gRPC server port | +| `HG_STORE_REST_PORT` | No | `8520` | `server.port` | REST API port | +| `HG_STORE_ROCKSDB_PROVIDER` | No | `rocksdb` | `rocksdb.provider` | `rocksdb` or `topling`; set `topling` explicitly with a Topling image | +| `HG_STORE_DATA_PATH` | No | Provider-specific path | `app.data-path` | `/hugegraph-store/storage` for RocksDB or `/hugegraph-store/topling-storage` for ToplingDB | +| `HG_STORE_ENFORCE_PROVIDER_MARKER` | No | `false` | Startup safety gate | Rejects an unmarked non-empty data path when `true`; enabled by Topling images | + +**Deprecated aliases** (still work but log a warning): + +| Deprecated | Use Instead | +|------------|-------------| +| `PD_ADDRESS` | `HG_STORE_PD_ADDRESS` | +| `GRPC_HOST` | `HG_STORE_GRPC_HOST` | +| `RAFT_ADDRESS` | `HG_STORE_RAFT_ADDRESS` | + +### Server Environment Variables + +| Variable | Required | Default | Maps To | Description | +|----------|----------|---------|-----------------------------|-------------| +| `HG_SERVER_BACKEND` | Yes | — | `backend` in `hugegraph.properties` | Storage backend (e.g. `hstore`) | +| `HG_SERVER_ROCKSDB_PROVIDER` | No | `rocksdb` | `rocksdb.provider` | RocksDB JNI provider (`rocksdb` or `topling`); set `topling` explicitly with a Topling image | +| `HG_SERVER_DATA_PATH` | No | Provider-specific image path | `rocksdb.data_path`, `rocksdb.wal_path` | RocksDB data root; standard uses `/hugegraph-server/rocksdb-data`, Topling uses `/hugegraph-server/topling-data` | +| `HG_SERVER_ENFORCE_PROVIDER_MARKER` | No | `false` | Startup safety gate | Rejects an unmarked non-empty data root when `true`; enabled by the Topling image | +| `HG_SERVER_PD_PEERS` | Yes | — | `pd.peers` | PD cluster addresses (e.g. `pd0:8686,pd1:8686,pd2:8686`) | +| `HG_SERVER_CLUSTER` | No | — | `cluster` in `rest-server.properties` | PD discovery application name; single-node Compose uses `hg` to match Hubble | +| `HG_SERVER_USE_PD` | No | — | `usePD` in `rest-server.properties` | Enables Server PD registration and discovery | +| `HG_SERVER_REST_URL` | No | — | `restserver.url` | Address registered with PD and used by clients | +| `HG_SERVER_MIN_FREE_MEMORY` | No | — | `restserver.min_free_memory` | Minimum free-memory guard in MB; local Compose uses `0` | +| `HG_SERVER_AUTH_TOKEN_SECRET` | No | generated in auth mode | `auth.token_secret` | Shared JWT secret for REST and embedded Gremlin authentication; explicit values must be at least 32 bytes | +| `STORE_REST` | No | — | Used by `wait-partition.sh` | Store REST endpoint for partition verification (e.g. `store0:8520`) | +| `PASSWORD` | No | — | Enables auth and sets `auth.admin_pa` | Initial administrator password; disabled init-store does not read it from stdin, but the entrypoint still applies it to the PD bootstrap path | +| `HG_SERVER_INIT_STORE_ENABLED` | No | `true` | `init_store.enabled` in `rest-server.properties` | Set `false` in PD/HStore deployments so init-store skips local backend and admin initialization | + +> **The built-in authenticator with `HG_SERVER_INIT_STORE_ENABLED=false` requires `usePD=true` and an HStore-backed `auth.graph_store`, unless `auth.remote_url` delegates auth elsewhere.** With init-store skipped, the server creates the built-in admin in PD metadata, and only an HStore auth graph uses the PD-backed auth manager that can read that account. init-store exits non-zero when the combination is unusable, rather than leaving a server nobody can log in to. A custom `auth.authenticator` is exempt because it manages its own identities. +> +> For a local RocksDB backend, init-store writes its completion marker below the provider data root at `.hugegraph-state/init_complete`. The marker therefore survives container recreation with the data volume. A standard RocksDB deployment migrates the legacy `docker/init_complete` marker once. HStore keeps the existing container-local marker behavior. A skipped run records nothing, so a later re-enable can still initialize. +> +> The entrypoint maps **`PASSWORD` to `auth.admin_pa`** before init-store runs. A disabled init-store does not read the password from standard input, but the PD startup path uses the explicit `auth.admin_pa` value when it first creates the administrator. Changing it later does not rotate an existing password. + +The single-node Compose files also accept these deployment-level overrides: + +| Variable | Default | Description | +|----------|---------|-------------| +| `HUGEGRAPH_SERVER_IMAGE` | `hugegraph/hugegraph:` (standalone); `hugegraph/server:` (HStore) | Server image reference | +| `HUGEGRAPH_SERVER_PULL_POLICY` | `missing` (`build` for dev) | Server pull policy | +| `HUBBLE_IMAGE` | `hugegraph/hubble:latest` | Complete Hubble image reference | +| `HUBBLE_PULL_POLICY` | `missing` | Hubble pull policy | +| `HUBBLE_PUBLISH_HOST` | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | +| `HUGEGRAPH_ADMIN_PASSWORD` | required (`docker/.env`) | Initial admin password; no public default is provided | +| `HUGEGRAPH_AUTH_TOKEN_SECRET` | generated | JWT signing secret; explicit values must be at least 32 bytes | + +When authentication is enabled and no token secret is supplied, the Server entrypoint generates a random secret and writes it to both authentication configurations. The value is reused on container restart while the container filesystem is preserved. To preserve tokens across container recreation, generate a compatible secret once and add it to the mode-600 `docker/.env`: + +```bash +( + set -euo pipefail + cd docker + secret_pattern='^[[:space:]]*(export[[:space:]]+)?HUGEGRAPH_AUTH_TOKEN_SECRET[[:space:]]*=' + secret_count="$(grep -Ec "${secret_pattern}" .env || true)" + case "${secret_count}" in + 0) + command -v openssl >/dev/null 2>&1 + token_secret="$(openssl rand -hex 32)" + if (( ${#token_secret} != 64 )); then + echo "Failed to generate a 64-character token secret" >&2 + exit 1 + fi + printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" "${token_secret}" >> .env + unset token_secret + echo "Generated HUGEGRAPH_AUTH_TOKEN_SECRET" + ;; + 1) + token_secret="$(sed -nE "s/${secret_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env)" + if (( ${#token_secret} < 32 )); then + echo "Existing token secret must contain at least 32 bytes" >&2 + exit 1 + fi + unset token_secret + echo "HUGEGRAPH_AUTH_TOKEN_SECRET already exists; reusing it" + ;; + *) + echo "Duplicate HUGEGRAPH_AUTH_TOKEN_SECRET entries; repair .env" >&2 + exit 1 + ;; + esac + chmod 600 .env +) +``` + +The entrypoint rejects shorter explicit values before changing either Server configuration file. + +**Deprecated aliases** (still work but log a warning): + +| Deprecated | Use Instead | +|------------|-------------| +| `BACKEND` | `HG_SERVER_BACKEND` | +| `PD_PEERS` | `HG_SERVER_PD_PEERS` | + +--- + Select Hubble independently: ```bash @@ -273,6 +423,75 @@ HUBBLE_PULL_POLICY=never \ docker compose -f docker-compose-hstore.yml up -d --wait ``` +### ToplingDB variants + +Build the Linux x86_64 Topling images from the shared Bake graph: + +```bash +RUNTIME_VARIANT=topling IMAGE_TAG=topling \ + docker buildx bake --file docker/bake.hcl +``` + +Run standalone HugeGraph with its isolated Topling data volume: + +```bash +HUGEGRAPH_SERVER_IMAGE=hugegraph/hugegraph:topling \ +HUGEGRAPH_SERVER_PULL_POLICY=always \ +HG_SERVER_ROCKSDB_PROVIDER=topling \ +HG_SERVER_DATA_PATH=/hugegraph-server/topling-data \ +HG_SERVER_ENFORCE_PROVIDER_MARKER=true \ +HUGEGRAPH_SERVER_VOLUME=server-topling-data \ +docker compose -f docker-compose.yml \ + up -d --wait +``` + +The same parameters work with a published image or a locally built image. The +standard `server-data` volume is not used; Topling data is stored in the +provider-specific `server-topling-data` volume. + +Run the minimal distributed 1+1+1 topology from source with Topling PD and +Store runtimes. The HStore Server remains standard: + +```bash +HUGEGRAPH_PD_IMAGE=local/hugegraph-pd:topling \ +HUGEGRAPH_PD_PULL_POLICY=never \ +HUGEGRAPH_PD_BUILD_TARGET=topling \ +HUGEGRAPH_PD_VOLUME=pd-topling-data \ +HG_PD_ROCKSDB_PROVIDER=topling \ +HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data \ +HG_PD_ENFORCE_PROVIDER_MARKER=true \ +HUGEGRAPH_STORE_IMAGE=local/hugegraph-store:topling \ +HUGEGRAPH_STORE_PULL_POLICY=never \ +HUGEGRAPH_STORE_BUILD_TARGET=topling \ +HUGEGRAPH_STORE_VOLUME=store-topling-data \ +HG_STORE_ROCKSDB_PROVIDER=topling \ +HG_STORE_DATA_PATH=/hugegraph-store/topling-storage \ +HG_STORE_ENFORCE_PROVIDER_MARKER=true \ +HUGEGRAPH_SERVER_IMAGE=hugegraph/server:dev \ +HUGEGRAPH_SERVER_PULL_POLICY=build \ +docker compose -f docker-compose-hstore.yml -f docker-compose.dev.yml \ + up -d --build --wait +``` + +For published images, replace the two local image values and `never` policies +with `hugegraph/{pd,store}:topling` and `always`. Set +`HUGEGRAPH_SERVER_IMAGE=hugegraph/server:topling` and +`HUGEGRAPH_SERVER_PULL_POLICY=always` for the matching HStore Server. The +standard `pd-data` and `store-data` volumes are not mounted. + +The HA reference uses the same parameters with +`docker-compose-3pd-3store-3server.yml`; set +`HUGEGRAPH_PD0_VOLUME`, `HUGEGRAPH_PD1_VOLUME`, `HUGEGRAPH_PD2_VOLUME`, +`HUGEGRAPH_STORE0_VOLUME`, `HUGEGRAPH_STORE1_VOLUME`, and +`HUGEGRAPH_STORE2_VOLUME` to their `*-topling-data` names. + +The generic Compose files inject image, provider, data-root, and volume +parameters; no Topling-specific Compose file is maintained. Each Topling owner +uses a provider-specific named volume and marker. Do not reuse a standard +RocksDB volume with a Topling image. See +[`docs/toplingdb/README.md`](../docs/toplingdb/README.md) for distribution +configuration, provider markers, restart checks, and troubleshooting. + ### Image build arguments and cache refresh Run image builds from the **repository root**. Direct Dockerfile builds and Bake use the same defaults: build the Server, PD, and Store distributions plus their dependencies (`-pl ... -am`), and reuse the OS package layer across source changes. diff --git a/docker/bake.hcl b/docker/bake.hcl index 2fc1cf4626..0713ba0325 100644 --- a/docker/bake.hcl +++ b/docker/bake.hcl @@ -31,6 +31,10 @@ variable "SOURCE_REVISION" { default = "local" } +variable "SOURCE_URL" { + default = "https://github.com/apache/hugegraph" +} + variable "IMAGE_TAG" { default = "local" } @@ -43,6 +47,14 @@ variable "EXPORT_CACHE" { default = false } +variable "RUNTIME_VARIANT" { + default = "standard" + validation { + condition = contains(["standard", "topling"], RUNTIME_VARIANT) + error_message = "RUNTIME_VARIANT must be standard or topling" + } +} + target "_common" { context = "." args = { @@ -50,8 +62,11 @@ target "_common" { MAVEN_PROJECTS = MAVEN_PROJECTS RUNTIME_DEPS_EPOCH = RUNTIME_DEPS_EPOCH SOURCE_REVISION = SOURCE_REVISION + SOURCE_REPOSITORY = SOURCE_URL } - platforms = [ + platforms = RUNTIME_VARIANT == "topling" ? [ + "linux/amd64", + ] : [ "linux/amd64", "linux/arm64", ] @@ -77,6 +92,7 @@ target "build-cache" { target "pd" { inherits = ["_common"] dockerfile = "hugegraph-pd/Dockerfile" + target = RUNTIME_VARIANT tags = ["hugegraph/pd:${IMAGE_TAG}"] output = ["type=docker"] cache-from = [ @@ -91,6 +107,7 @@ target "pd" { target "store" { inherits = ["_common"] dockerfile = "hugegraph-store/Dockerfile" + target = RUNTIME_VARIANT tags = ["hugegraph/store:${IMAGE_TAG}"] output = ["type=docker"] cache-from = [ @@ -119,6 +136,7 @@ target "server-hstore" { target "server-standalone" { inherits = ["_common"] dockerfile = "hugegraph-server/Dockerfile" + target = RUNTIME_VARIANT tags = ["hugegraph/hugegraph:${IMAGE_TAG}"] output = ["type=docker"] cache-from = [ diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index 999f2126fb..f12e1e628e 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -23,17 +23,23 @@ networks: volumes: hg-pd0-data: + hg-pd0-topling-data: hg-pd1-data: + hg-pd1-topling-data: hg-pd2-data: + hg-pd2-topling-data: hg-store0-data: + hg-store0-topling-data: hg-store1-data: + hg-store1-topling-data: hg-store2-data: + hg-store2-topling-data: hubble-data: # ── Shared service defaults ────────────────────────────────────────── x-pd-common: &pd-common - image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + image: ${HUGEGRAPH_PD_IMAGE:-hugegraph/pd:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_PD_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] healthcheck: @@ -44,8 +50,8 @@ x-pd-common: &pd-common start_period: 120s x-store-common: &store-common - image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + image: ${HUGEGRAPH_STORE_IMAGE:-hugegraph/store:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_STORE_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] depends_on: @@ -77,8 +83,8 @@ x-server-environment: &server-environment PD_AUTH_PASSWORD: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} x-server-common: &server-common - image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-missing} restart: unless-stopped networks: [hg-net] depends_on: @@ -110,12 +116,14 @@ services: HG_PD_RAFT_ADDRESS: pd0:8610 HG_PD_RAFT_PEERS_LIST: pd0:8610,pd1:8610,pd2:8610 HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_INITIAL_STORE_COUNT: 3 HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: ["8620:8620", "8686:8686"] volumes: - - hg-pd0-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD0_VOLUME:-hg-pd0-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} pd1: <<: *pd-common @@ -129,12 +137,14 @@ services: HG_PD_RAFT_ADDRESS: pd1:8610 HG_PD_RAFT_PEERS_LIST: pd0:8610,pd1:8610,pd2:8610 HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_INITIAL_STORE_COUNT: 3 HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: ["8621:8620", "8687:8686"] volumes: - - hg-pd1-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD1_VOLUME:-hg-pd1-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} pd2: <<: *pd-common @@ -148,12 +158,14 @@ services: HG_PD_RAFT_ADDRESS: pd2:8610 HG_PD_RAFT_PEERS_LIST: pd0:8610,pd1:8610,pd2:8610 HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_INITIAL_STORE_COUNT: 3 HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: ["8622:8620", "8688:8686"] volumes: - - hg-pd2-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD2_VOLUME:-hg-pd2-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} # --- Store cluster (3 nodes) --- store0: @@ -166,10 +178,12 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store0:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: ["8500:8500", "8510:8510", "8520:8520"] volumes: - - hg-store0-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE0_VOLUME:-hg-store0-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} store1: <<: *store-common @@ -181,10 +195,12 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store1:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: ["8501:8500", "8511:8510", "8521:8520"] volumes: - - hg-store1-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE1_VOLUME:-hg-store1-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} store2: <<: *store-common @@ -196,10 +212,12 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store2:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: ["8502:8500", "8512:8510", "8522:8520"] volumes: - - hg-store2-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE2_VOLUME:-hg-store2-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} # --- Server cluster (3 nodes) --- server0: diff --git a/docker/docker-compose-hstore.yml b/docker/docker-compose-hstore.yml index 0fcc3697f6..948354ff2f 100644 --- a/docker/docker-compose-hstore.yml +++ b/docker/docker-compose-hstore.yml @@ -22,13 +22,15 @@ networks: volumes: pd-data: + pd-topling-data: store-data: + store-topling-data: hubble-data: services: pd: - image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_PD_IMAGE:-hugegraph/pd:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_PD_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] environment: @@ -38,12 +40,14 @@ services: HG_PD_RAFT_ADDRESS: pd:8610 HG_PD_RAFT_PEERS_LIST: pd:8610 HG_PD_INITIAL_STORE_LIST: store:8500 - HG_PD_DATA_PATH: /hugegraph-pd/pd_data + HG_PD_DATA_PATH: ${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} + HG_PD_ROCKSDB_PROVIDER: ${HG_PD_ROCKSDB_PROVIDER:-rocksdb} + HG_PD_ENFORCE_PROVIDER_MARKER: ${HG_PD_ENFORCE_PROVIDER_MARKER:-false} HG_PD_AUTH_SECRET_KEY: ${HG_PD_AUTH_SECRET_KEY:?set HG_PD_AUTH_SECRET_KEY in .env; see docker/README.md} ports: - "8620:8620" volumes: - - pd-data:/hugegraph-pd/pd_data + - ${HUGEGRAPH_PD_VOLUME:-pd-data}:${HG_PD_DATA_PATH:-/hugegraph-pd/pd_data} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8620/v1/health >/dev/null"] interval: 10s @@ -52,8 +56,8 @@ services: start_period: 30s store: - image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_STORE_IMAGE:-hugegraph/store:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_STORE_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] depends_on: @@ -65,11 +69,13 @@ services: HG_STORE_GRPC_PORT: "8500" HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store:8510 - HG_STORE_DATA_PATH: /hugegraph-store/storage + HG_STORE_DATA_PATH: ${HG_STORE_DATA_PATH:-/hugegraph-store/storage} + HG_STORE_ROCKSDB_PROVIDER: ${HG_STORE_ROCKSDB_PROVIDER:-rocksdb} + HG_STORE_ENFORCE_PROVIDER_MARKER: ${HG_STORE_ENFORCE_PROVIDER_MARKER:-false} ports: - "8520:8520" volumes: - - store-data:/hugegraph-store/storage + - ${HUGEGRAPH_STORE_VOLUME:-store-data}:${HG_STORE_DATA_PATH:-/hugegraph-store/storage} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8520/v1/health >/dev/null"] interval: 10s @@ -78,8 +84,8 @@ services: start_period: 60s server: - image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-missing} restart: unless-stopped networks: [hg-net] depends_on: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 4da9c32e19..3de4a7d8ee 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -17,26 +17,28 @@ services: pd: - image: hugegraph/pd:dev - pull_policy: build + image: ${HUGEGRAPH_PD_IMAGE:-hugegraph/pd:dev} + pull_policy: ${HUGEGRAPH_PD_PULL_POLICY:-build} build: context: .. dockerfile: hugegraph-pd/Dockerfile + target: ${HUGEGRAPH_PD_BUILD_TARGET:-standard} healthcheck: start_period: 20s store: - image: hugegraph/store:dev - pull_policy: build + image: ${HUGEGRAPH_STORE_IMAGE:-hugegraph/store:dev} + pull_policy: ${HUGEGRAPH_STORE_PULL_POLICY:-build} build: context: .. dockerfile: hugegraph-store/Dockerfile + target: ${HUGEGRAPH_STORE_BUILD_TARGET:-standard} healthcheck: start_period: 30s server: - image: hugegraph/server:dev - pull_policy: build + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/server:dev} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-build} build: context: .. dockerfile: hugegraph-server/Dockerfile-hstore diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d85cf5b1c6..171e2e9835 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,18 +22,22 @@ networks: volumes: server-data: + server-topling-data: hubble-data: services: server: - image: hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest} - pull_policy: ${HUGEGRAPH_PULL_POLICY:-missing} + image: ${HUGEGRAPH_SERVER_IMAGE:-hugegraph/hugegraph:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUGEGRAPH_SERVER_PULL_POLICY:-${HUGEGRAPH_PULL_POLICY:-missing}} restart: unless-stopped networks: [hg-net] environment: PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} HG_SERVER_MIN_FREE_MEMORY: "0" + HG_SERVER_ROCKSDB_PROVIDER: ${HG_SERVER_ROCKSDB_PROVIDER:-rocksdb} + HG_SERVER_DATA_PATH: ${HG_SERVER_DATA_PATH:-/hugegraph-server/rocksdb-data} + HG_SERVER_ENFORCE_PROVIDER_MARKER: ${HG_SERVER_ENFORCE_PROVIDER_MARKER:-false} # Unset-only default: a host value overrides it, an absent one # renders the entrypoint default. ":-" would turn an empty host # value into 120 silently, which the entrypoint rejects on purpose. @@ -41,7 +45,7 @@ services: ports: - "8080:8080" volumes: - - server-data:/hugegraph-server/rocksdb-data + - ${HUGEGRAPH_SERVER_VOLUME:-server-data}:${HG_SERVER_DATA_PATH:-/hugegraph-server/rocksdb-data} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null"] interval: 10s diff --git a/docker/test-compose.sh b/docker/test-compose.sh index 54418d413a..2dc3e2854a 100644 --- a/docker/test-compose.sh +++ b/docker/test-compose.sh @@ -52,6 +52,66 @@ compose_auth() { docker compose "$@" } +topling_env() { + env HG_PD_AUTH_SECRET_KEY="${PD_SECRET}" \ + HUGEGRAPH_VERSION="${VERSION}" \ + HUBBLE_IMAGE="${RENDER_HUBBLE_IMAGE}" \ + HUGEGRAPH_ADMIN_PASSWORD="${PASSWORD}" \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${SECRET}" \ + HUGEGRAPH_PD_IMAGE="local/hugegraph-pd:topling" \ + HUGEGRAPH_PD_PULL_POLICY="never" \ + HUGEGRAPH_PD_BUILD_TARGET="topling" \ + HUGEGRAPH_PD_VOLUME="pd-topling-data" \ + HUGEGRAPH_PD0_VOLUME="hg-pd0-topling-data" \ + HUGEGRAPH_PD1_VOLUME="hg-pd1-topling-data" \ + HUGEGRAPH_PD2_VOLUME="hg-pd2-topling-data" \ + HG_PD_ROCKSDB_PROVIDER="topling" \ + HG_PD_DATA_PATH="/hugegraph-pd/topling-pd-data" \ + HG_PD_ENFORCE_PROVIDER_MARKER="true" \ + HUGEGRAPH_STORE_IMAGE="local/hugegraph-store:topling" \ + HUGEGRAPH_STORE_PULL_POLICY="never" \ + HUGEGRAPH_STORE_BUILD_TARGET="topling" \ + HUGEGRAPH_STORE_VOLUME="store-topling-data" \ + HUGEGRAPH_STORE0_VOLUME="hg-store0-topling-data" \ + HUGEGRAPH_STORE1_VOLUME="hg-store1-topling-data" \ + HUGEGRAPH_STORE2_VOLUME="hg-store2-topling-data" \ + HG_STORE_ROCKSDB_PROVIDER="topling" \ + HG_STORE_DATA_PATH="/hugegraph-store/topling-storage" \ + HG_STORE_ENFORCE_PROVIDER_MARKER="true" \ + "$@" +} + +render_topling_standalone() { + local output="$1" + shift + topling_env \ + env HUGEGRAPH_SERVER_IMAGE="local/hugegraph:topling" \ + HUGEGRAPH_SERVER_PULL_POLICY="never" \ + HUGEGRAPH_SERVER_VOLUME="server-topling-data" \ + HG_SERVER_ROCKSDB_PROVIDER="topling" \ + HG_SERVER_DATA_PATH="/hugegraph-server/topling-data" \ + HG_SERVER_ENFORCE_PROVIDER_MARKER="true" \ + docker compose "$@" config --format json > "${output}" +} + +render_topling_hstore() { + local output="$1" + shift + topling_env \ + env HUGEGRAPH_SERVER_IMAGE="hugegraph/server:${VERSION}" \ + HUGEGRAPH_SERVER_PULL_POLICY="missing" \ + docker compose "$@" config --format json > "${output}" +} + +render_topling_ha() { + local output="$1" + shift + topling_env \ + env HUGEGRAPH_SERVER_IMAGE="hugegraph/server:${VERSION}" \ + HUGEGRAPH_SERVER_PULL_POLICY="missing" \ + docker compose "$@" config --format json > "${output}" +} + render() { local output="$1" shift @@ -313,6 +373,60 @@ assert_dev_override() { ' "${override}" >/dev/null } +assert_topling_standalone() { + local rendered="$1" + jq -e ' + .name == "hugegraph-standalone" and + .services.server.image == "local/hugegraph:topling" and + .services.server.pull_policy == "never" and + .services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == "topling" and + .services.server.environment.HG_SERVER_ENFORCE_PROVIDER_MARKER == "true" and + .services.server.environment.HG_SERVER_DATA_PATH == "/hugegraph-server/topling-data" and + (.services.server.volumes | length == 1) and + .services.server.volumes[0].source == "server-topling-data" and + .services.server.volumes[0].target == "/hugegraph-server/topling-data" + ' "${rendered}" >/dev/null +} + +assert_topling_hstore() { + local rendered="$1" + jq -e ' + .name == "hugegraph-hstore" and + .services.pd.image == "local/hugegraph-pd:topling" and + .services.store.image == "local/hugegraph-store:topling" and + .services.pd.pull_policy == "never" and + .services.store.pull_policy == "never" and + .services.pd.environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + .services.store.environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + (.services.pd.volumes | length == 1) and + (.services.store.volumes | length == 1) and + .services.pd.volumes[0].source == "pd-topling-data" and + .services.store.volumes[0].source == "store-topling-data" and + .services.server.environment.HG_SERVER_BACKEND == "hstore" and + (.services.server.environment.HG_SERVER_ROCKSDB_PROVIDER == null) + ' "${rendered}" >/dev/null +} + +assert_topling_ha() { + local rendered="$1" + jq -e ' + .name == "hugegraph-3x3" and + ([.services.pd0, .services.pd1, .services.pd2] | + all(.image == "local/hugegraph-pd:topling" and + .pull_policy == "never" and + .environment.HG_PD_ROCKSDB_PROVIDER == "topling" and + (.volumes | length == 1))) and + ([.services.store0, .services.store1, .services.store2] | + all(.image == "local/hugegraph-store:topling" and + .pull_policy == "never" and + .environment.HG_STORE_ROCKSDB_PROVIDER == "topling" and + (.volumes | length == 1))) and + ([.services.server0, .services.server1, .services.server2] | + all(.environment.HG_SERVER_BACKEND == "hstore" and + (.environment.HG_SERVER_ROCKSDB_PROVIDER == null))) + ' "${rendered}" >/dev/null +} + cleanup() { if [[ -n "${ACTIVE_PROJECT}" ]]; then compose_active down -v --remove-orphans >/dev/null 2>&1 || true @@ -400,12 +514,22 @@ run_render() { -f "${DOCKER_DIR}/docker-compose.dev.yml" render "${RENDER_DIR}/override.json" \ -f "${DOCKER_DIR}/docker-compose.dev.yml" + render_topling_standalone "${RENDER_DIR}/topling-standalone.json" \ + -f "${DOCKER_DIR}/docker-compose.yml" + render_topling_hstore "${RENDER_DIR}/topling-hstore.json" \ + -f "${DOCKER_DIR}/docker-compose-hstore.yml" \ + -f "${DOCKER_DIR}/docker-compose.dev.yml" + render_topling_ha "${RENDER_DIR}/topling-ha.json" \ + -f "${DOCKER_DIR}/docker-compose-3pd-3store-3server.yml" assert_standalone "${RENDER_DIR}/standalone.json" assert_hstore "${RENDER_DIR}/hstore.json" assert_ha "${RENDER_DIR}/ha.json" assert_dev_override "${RENDER_DIR}/dev.json" \ "${RENDER_DIR}/override.json" + assert_topling_standalone "${RENDER_DIR}/topling-standalone.json" + assert_topling_hstore "${RENDER_DIR}/topling-hstore.json" + assert_topling_ha "${RENDER_DIR}/topling-ha.json" # An absent host value renders the entrypoint default in every topology and # a host value overrides it, so the container budget can be lowered to meet diff --git a/docs/toplingdb/README.md b/docs/toplingdb/README.md new file mode 100644 index 0000000000..ef17b62ba5 --- /dev/null +++ b/docs/toplingdb/README.md @@ -0,0 +1,33 @@ +# ToplingDB Documentation + +ToplingDB is an optional RocksDB-compatible runtime for HugeGraph. Select it +when a HugeGraph process owns a local RocksDB database: + +| Deployment | Process that selects ToplingDB | +|---|---| +| Standalone | HugeGraph Server | +| Distributed HStore | PD and Store | +| HStore-backed Server | None; it is a remote client | + +Start with one of these paths: + +- [Quickstart](toplingdb-quickstart.md) explains how to build a Topling + distribution, select the provider, keep its data separate, start the + component, and return to standard RocksDB safely. +- [Developer guide](toplingdb-development.md) covers the source layout, + distribution contract, tests, CI, Docker status, and native runtime work. +- [Issue illustrations](images/issues/v1/README.md) explains the scope and + core issues tracked by [Summary #240](https://github.com/hugegraph/hugegraph/issues/240). + +Use the reference guides when you need more detail: + +- [Architecture and configuration](toplingdb.md) +- [HStore integration](toplingdb-hstore-integration.md) +- [Operations](toplingdb-operations.md) +- [Troubleshooting](toplingdb-troubleshooting.md) +- [Security](toplingdb-security.md) + +ToplingDB selection happens only at process startup. Changing +`rocksdb.provider` does not migrate or convert an existing data directory. +Standard RocksDB and ToplingDB must use separate data directories unless you +restore a verified, compatible snapshot into an empty directory. diff --git a/docs/toplingdb/images/issues/v1/README.md b/docs/toplingdb/images/issues/v1/README.md new file mode 100644 index 0000000000..c7c9560ec6 --- /dev/null +++ b/docs/toplingdb/images/issues/v1/README.md @@ -0,0 +1,29 @@ +# ToplingDB issue 图片 + +对应 [Summary #240](https://github.com/hugegraph/hugegraph/issues/240) 与 TP 主线 10 个子 issue。 + +使用内置 `image_gen` 生成；完整提示词见 [prompts.json](prompts.json)。图示说明问题和目标，不代替 issue 正文中的验证证据。 + +| Issue | 主题 | 图片 | +| --- | --- | --- | +| [#240](https://github.com/hugegraph/hugegraph/issues/240) | 全局总览 | [查看图片](issue-240-overview.png) | +| [#250](https://github.com/hugegraph/hugegraph/issues/250) | 统一 provider 来源 | [查看图片](issue-250-provider-source.png) | +| [#251](https://github.com/hugegraph/hugegraph/issues/251) | 实际 graphs 目录 | [查看图片](issue-251-graphs-directory.png) | +| [#253](https://github.com/hugegraph/hugegraph/issues/253) | 所有图的数据根校验 | [查看图片](issue-253-all-data-roots.png) | +| [#254](https://github.com/hugegraph/hugegraph/issues/254) | 多 key truncate 覆盖 | [查看图片](issue-254-truncate-coverage.png) | +| [#255](https://github.com/hugegraph/hugegraph/issues/255) | native 诊断分类 | [查看图片](issue-255-diagnostic-gates.png) | +| [#249](https://github.com/hugegraph/hugegraph/issues/249) | WAL 恢复失败安全 | [查看图片](issue-249-wal-recovery.png) | +| [#248](https://github.com/hugegraph/hugegraph/issues/248) | 重启后不可见的调查 | [查看图片](issue-248-restart-investigation.png) | +| [#212](https://github.com/hugegraph/hugegraph/issues/212) | DB / CF 生命周期 | [查看图片](issue-212-cf-lifecycle.png) | +| [#213](https://github.com/hugegraph/hugegraph/issues/213) | JNI 可追溯交付 | [查看图片](issue-213-jni-provenance.png) | +| [#252](https://github.com/hugegraph/hugegraph/issues/252) | 固定 workload 对照 | [查看图片](issue-252-benchmark-plan.png) | + +## 总览 + +![TP 主线与通用独立跟进两类问题的总览](issue-240-overview.png) + +## 使用说明 + +- 每张图与一个 issue 对应，点击图片可查看原尺寸。 +- 已确认缺口、待归因线索、历史验证和目标状态分别表达。 +- 非 TP 专属的通用子 issue 不在此图片集范围内。 diff --git a/docs/toplingdb/images/issues/v1/assets.json b/docs/toplingdb/images/issues/v1/assets.json new file mode 100644 index 0000000000..ddbde5adb6 --- /dev/null +++ b/docs/toplingdb/images/issues/v1/assets.json @@ -0,0 +1,148 @@ +{ + "mode": "built-in image_gen", + "assets": [ + { + "issue": 240, + "title": "全局总览", + "filename": "issue-240-overview.png", + "alt": "TP 主线与通用独立跟进两类问题的总览", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/240", + "width": 1672, + "height": 941, + "bytes": 1217558, + "sha256": "a14158c009d8c54ef65e87463009def46b4bf61d4ca9c54cc0aeeecaa7858b8a", + "qa": "PASS: visually checked issue number, legible text, current vs target distinction and technical relationships", + "revised": false + }, + { + "issue": 250, + "title": "统一 provider 来源", + "filename": "issue-250-provider-source.png", + "alt": "JNI runtime 与 Java provider 的配置分叉", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/250", + "width": 1672, + "height": 941, + "bytes": 1186616, + "sha256": "31fd493d918a2fec2174ff04fd86e402cc024ad7b35b24437c50203e79dba9f4", + "qa": "PASS: visually checked issue number, legible text, current vs target distinction and technical relationships", + "revised": false + }, + { + "issue": 251, + "title": "实际 graphs 目录", + "filename": "issue-251-graphs-directory.png", + "alt": "selector 默认目录和实际图配置目录错位", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/251", + "width": 1672, + "height": 941, + "bytes": 1212922, + "sha256": "06cf01e644fcdcc02059d6f17ea8c29eac3a6597a0d5ec36b47f98e16c4227a9", + "qa": "PASS: visually checked issue number, legible text, current vs target distinction and technical relationships", + "revised": false + }, + { + "issue": 253, + "title": "所有图的数据根校验", + "filename": "issue-253-all-data-roots.png", + "alt": "默认数据根与额外图数据根的校验覆盖", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/253", + "width": 1672, + "height": 941, + "bytes": 1354750, + "sha256": "8e8cbd551c0f97e777991b1ec34a1c380fa08900459de129de01cd7b1491be0f", + "qa": "PASS: visually checked issue number, legible text, current vs target distinction and technical relationships", + "revised": true + }, + { + "issue": 254, + "title": "多 key truncate 覆盖", + "filename": "issue-254-truncate-coverage.png", + "alt": "单 key 测试与多 key 范围删除回归的差别", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/254", + "width": 1672, + "height": 941, + "bytes": 1137409, + "sha256": "3f24d7bfc7a0ecee679ea7b757d15a7cfede312a64b2978641dc0f934f28347f", + "qa": "通过；[k1,k3) 与单独 delete k3 分开表示；目标与现有单测分开；覆盖缺口不等于清理失败。", + "revised": false + }, + { + "issue": 255, + "title": "native 诊断分类", + "filename": "issue-255-diagnostic-gates.png", + "alt": "已知 CF 断言与其他 native 诊断错误的分类", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/255", + "width": 1672, + "height": 941, + "bytes": 1140750, + "sha256": "92affa03ad76b3c7d0717381ad0e8fb8ab8a251f32bbe73431bdac0b3f78ab4f", + "qa": "通过；当前全部非阻塞与目标分类诊断区分；未声称已修复。", + "revised": false + }, + { + "issue": 249, + "title": "WAL 恢复失败安全", + "filename": "issue-249-wal-recovery.png", + "alt": "旧 WAL 隔离、checkpoint tail 暂存和校验发布", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/249", + "width": 1672, + "height": 941, + "bytes": 1226490, + "sha256": "e353a54d9e7408ee1e9aa05bc96be0d24b8ace769a2367012aed0a5a75c95c6e", + "qa": "通过；恢复源保留，旧WAL隔离、新WAL由checkpoint/staging发布；明确目标流程与待收口，未声称原子性或native特有问题。", + "revised": true + }, + { + "issue": 248, + "title": "重启后不可见的调查", + "filename": "issue-248-restart-investigation.png", + "alt": "首次与第二次 Server 重启现象，以及尚未确认的根因", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/248", + "width": 1672, + "height": 941, + "bytes": 1375350, + "sha256": "48a1903819332058c90f7011ca1427d9365b3bee8a004191ca91ed4f01931d5b", + "qa": "通过；Server重启历史现象与v2对照清晰；物理数据状态待核实，根因假设待验证。", + "revised": true + }, + { + "issue": 212, + "title": "DB / CF 生命周期", + "filename": "issue-212-cf-lifecycle.png", + "alt": "历史生命周期修复与当前残余 native 告警的区别", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/212", + "width": 1672, + "height": 941, + "bytes": 1176499, + "sha256": "5c7528af7bb0c1fbe3292efecc357d1ac4ee46f823137f9d4e4d15b4261e84c3", + "qa": "通过：历史修复与当前残余归因分栏，旧 SHA 通过不等于当前闭环；无整体完成声明。", + "revised": false + }, + { + "issue": 213, + "title": "JNI 可追溯交付", + "filename": "issue-213-jni-provenance.png", + "alt": "候选 JNI 与正式可追溯交付链的区别", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/213", + "width": 1672, + "height": 941, + "bytes": 1113499, + "sha256": "4628fcaace585ffaab10452d1f2a4e5d8e8cd8c41ac959fee7f451b98e450728", + "qa": "通过：候选 SHA-256 与待完成目标交付链分开，无虚构哈希、版本或完整来源证明。", + "revised": false + }, + { + "issue": 252, + "title": "固定 workload 对照", + "filename": "issue-252-benchmark-plan.png", + "alt": "同条件 RocksDB / Topling 对照的待执行计划", + "issue_url": "https://github.com/hugegraph/hugegraph/issues/252", + "width": 1672, + "height": 941, + "bytes": 1102773, + "sha256": "7aa36acb5c53fc58a73e3936771405084a1bc5d7c7e3c44e7de7ae8b99236de8", + "qa": "通过：待执行状态明确，两条同等负载路径与三轮空报告，无虚构性能数字或胜负。蓝灰配色与提示略有偏差但不影响技术含义。", + "revised": false + } + ] +} diff --git a/docs/toplingdb/images/issues/v1/index.html b/docs/toplingdb/images/issues/v1/index.html new file mode 100644 index 0000000000..71870c0327 --- /dev/null +++ b/docs/toplingdb/images/issues/v1/index.html @@ -0,0 +1,19 @@ + +ToplingDB Issue Images

ToplingDB issue 图片

全局总览 + TP 主线 10 项 · 点击图片查看原图 · Summary #240 · 生成提示词

JNI runtime 与 Java provider 的配置分叉
#250统一 provider 来源
selector 默认目录和实际图配置目录错位
#251实际 graphs 目录
默认数据根与额外图数据根的校验覆盖
#253所有图的数据根校验
单 key 测试与多 key 范围删除回归的差别
#254多 key truncate 覆盖
已知 CF 断言与其他 native 诊断错误的分类
#255native 诊断分类
旧 WAL 隔离、checkpoint tail 暂存和校验发布
#249WAL 恢复失败安全
首次与第二次 Server 重启现象，以及尚未确认的根因
#248重启后不可见的调查
历史生命周期修复与当前残余 native 告警的区别
#212DB / CF 生命周期
候选 JNI 与正式可追溯交付链的区别
#213JNI 可追溯交付
同条件 RocksDB / Topling 对照的待执行计划
#252固定 workload 对照
diff --git a/docs/toplingdb/images/issues/v1/issue-212-cf-lifecycle.png b/docs/toplingdb/images/issues/v1/issue-212-cf-lifecycle.png new file mode 100644 index 0000000000..b9df678a52 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-212-cf-lifecycle.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-213-jni-provenance.png b/docs/toplingdb/images/issues/v1/issue-213-jni-provenance.png new file mode 100644 index 0000000000..3e3a1489ba Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-213-jni-provenance.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-240-overview.png b/docs/toplingdb/images/issues/v1/issue-240-overview.png new file mode 100644 index 0000000000..4157ea12ee Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-240-overview.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-248-restart-investigation.png b/docs/toplingdb/images/issues/v1/issue-248-restart-investigation.png new file mode 100644 index 0000000000..dc1a98f813 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-248-restart-investigation.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-249-wal-recovery.png b/docs/toplingdb/images/issues/v1/issue-249-wal-recovery.png new file mode 100644 index 0000000000..c5bd2e624c Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-249-wal-recovery.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-250-provider-source.png b/docs/toplingdb/images/issues/v1/issue-250-provider-source.png new file mode 100644 index 0000000000..8b58d5d102 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-250-provider-source.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-251-graphs-directory.png b/docs/toplingdb/images/issues/v1/issue-251-graphs-directory.png new file mode 100644 index 0000000000..3ab6c1bcd2 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-251-graphs-directory.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-252-benchmark-plan.png b/docs/toplingdb/images/issues/v1/issue-252-benchmark-plan.png new file mode 100644 index 0000000000..b6ef9be312 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-252-benchmark-plan.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-253-all-data-roots.png b/docs/toplingdb/images/issues/v1/issue-253-all-data-roots.png new file mode 100644 index 0000000000..d5954a92f7 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-253-all-data-roots.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-254-truncate-coverage.png b/docs/toplingdb/images/issues/v1/issue-254-truncate-coverage.png new file mode 100644 index 0000000000..ecc6993db9 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-254-truncate-coverage.png differ diff --git a/docs/toplingdb/images/issues/v1/issue-255-diagnostic-gates.png b/docs/toplingdb/images/issues/v1/issue-255-diagnostic-gates.png new file mode 100644 index 0000000000..5f23d02215 Binary files /dev/null and b/docs/toplingdb/images/issues/v1/issue-255-diagnostic-gates.png differ diff --git a/docs/toplingdb/images/issues/v1/prompts.json b/docs/toplingdb/images/issues/v1/prompts.json new file mode 100644 index 0000000000..56fe710e14 --- /dev/null +++ b/docs/toplingdb/images/issues/v1/prompts.json @@ -0,0 +1,100 @@ +{ + "mode": "built-in image_gen", + "use_case": "infographic-diagram", + "assets": [ + { + "issue": 240, + "slug": "overview", + "filename": "issue-240-overview.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: An architectural issue map with a clear central TP pipeline and a separate muted strip for general platform work. Title text: \"#240\" and \"ToplingDB 适配与实测\". Main flow labels: \"配置\" → \"启动 / JNI\" → \"DB / CF\" → \"验证\" → \"交付\". Under it, four visually grouped TP work areas labeled \"选择与隔离 #250 #251 #253\", \"覆盖与诊断 #254 #255\", \"恢复与归因 #249 #248 #212\", \"交付与对照 #213 #252\". Label this main region \"TP 主线 · 10 项\". A visually secondary independent horizontal lane labeled \"通用问题 · 11 项\" contains compact labels \"Schema / Cache\", \"连接与资源\", \"图快照\", \"部署\" and the explicit note \"ignore ≠ 已解决\". Do not draw this gray lane as a prerequisite blocking the TP path. Footer: \"Mac：开发与最小验证 | Linux：实测与性能\". This is a scope map, not a completion chart. No progress percentages or green all-passed stamps.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/240", + "revision_prompts": [] + }, + { + "issue": 250, + "slug": "provider-source", + "filename": "issue-250-provider-source.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: Two parallel decisions inside the same Server that disagree. Title: \"#250\" and \"Provider 来源必须一致\". Current-state region clearly labeled \"当前问题\". One lane starts at a small environment-variable card labeled \"ENV: topling\", goes to a runtime module \"JNI: Topling\". A second lane starts at a configuration-file card \"Config: rocksdb\", goes to a Java module \"Java: RocksDB\". Connect their end nodes with a restrained mismatch mark and label \"状态分叉\". Show consequence as a small CF module labeled \"truncate 路径不一致\"; do not depict data destruction. A separate desired-state region labeled \"目标\" shows one configuration hub labeled \"唯一有效 provider\" branching consistently to \"JNI\" and \"Java\". Footer label: \"已复现配置分叉 · native 行为待验证\". Keep environment/config and native/Java differences explicit; this is the direct-launch override case, not a claim that Docker default is broken.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/250", + "revision_prompts": [] + }, + { + "issue": 251, + "slug": "graphs-directory", + "filename": "issue-251-graphs-directory.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: An incorrect directory lookup versus the configured directory. Title: \"#251\" and \"读取实际生效的图目录\". Use a configuration card labeled \"graphs=./custom-graphs\" as the authoritative input. Under \"当前问题\", show a selector magnifier searching a folder \"conf/graphs\" and selecting a gray chip \"rocksdb\". In parallel, show the Java graph loader reading a blue folder \"custom-graphs\" with a small file labeled \"provider=topling\". Highlight the diverging paths with the label \"目录错位\". Under \"目标\", the same actual configuration feeds both nodes \"Selector\" and \"Java\" through one shared directory. Footer \"自定义目录不应静默选择标准 runtime\". Do not draw missing config as an error raised, because current selector exits successfully with the wrong selection.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/251", + "revision_prompts": [] + }, + { + "issue": 253, + "slug": "all-data-roots", + "filename": "issue-253-all-data-roots.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: Provider marker checking covers only the default graph, leaving an extra graph's data root outside the check. Title: \"#253\" and \"校验每个图的数据根\". Main current-state diagram labeled \"当前问题\": two graph cards \"默认图\" and \"额外图\". Default graph points through a blue guard/checkpoint labeled \"marker 校验\" to database root \"/tp-data\". Extra graph points around that guard on an amber dashed path to root \"/other-data\", with a small existing tag \"provider=rocksdb\" attached. On the extra graph card add \"provider=topling\". Label bypass \"未进入校验\". Do not depict actual data corruption or deletion. A small desired-state diagram labeled \"目标\" routes both graph roots through a common validation gate before a database-open icon labeled \"打开数据库\". Footer \"已确认校验覆盖遗漏 · 未打开跨 provider 数据\". The image should make the call-coverage gap clear, not suggest the marker helper itself is broken.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/253", + "revision_prompts": [ + "Edit this technical infographic, preserving its typography, layout, colors, title #253 and footer. Correct ONLY these technical relationships: In the LEFT current-state panel, the extra graph card must contain only provider=topling. Move provider=rocksdb from that card to a tag attached directly below the /other-data database root. The dashed amber extra-graph arrow must run straight horizontally below the marker 校验 gate, from extra graph directly to /other-data, NEVER touching or entering the gate; place 未进入校验 above this horizontal bypass arrow. In the RIGHT target panel, remove provider=rocksdb from the extra graph card entirely, keep only provider=topling, preserve both graphs entering the common marker 校验 gate before 打开数据库. Everything else unchanged. Accurate Chinese labels; no added text." + ] + }, + { + "issue": 254, + "slug": "truncate-coverage", + "filename": "issue-254-truncate-coverage.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: Missing multi-key coverage for the TP clearTables range-delete branch. Title: \"#254\" and \"覆盖多 key 的 truncate\". Two clear panels: \"现有单测\" contains a single CF container with one key \"k1\" and the label \"first = last\", with a bypass arrow labeled \"跳过 deleteRange\". \"待补回归\" contains the same intact CF container holding \"k1\", \"k2\", \"k3\". Depict a range bracket \"[k1, k3)\" plus a separate action on \"k3\" to communicate range deletion followed by deleting the last key. Below, the intended validation sequence is labeled \"旧数据为空\" → \"CF 保留\" → \"新写入可读\", all under \"目标\", not as already passed. Add a small runtime badge \"Topling JNI\" and footer \"覆盖缺口 ≠ 已确认清理失败\". No claim that Topling has never had real lifecycle tests.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/254", + "revision_prompts": [] + }, + { + "issue": 255, + "slug": "diagnostic-gates", + "filename": "issue-255-diagnostic-gates.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: A CI diagnostic incorrectly funnels every kind of error into one known-assertion bucket. Title: \"#255\" and \"已知断言不能掩盖其他失败\". Current panel labeled \"当前问题\": four compact error cards \"已知 CF 断言\", \"配置失败\", \"JNI 加载失败\", \"读写失败\" all converge to a single amber box \"全部非阻塞\", with a small tag \"统一归因 #212\". Desired panel labeled \"目标\": a classifier gate sends only \"已知 CF 断言\" to \"保留诊断\", and sends the other errors to a red stop/status box \"检查失败\". Use visible split arrows and restrained failure indicators. Footer \"按实际错误分类 · 保留真实原因\". This explains one diagnostic step; do not claim all CI tests or all runtime checks are ineffective.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/255", + "revision_prompts": [] + }, + { + "issue": 249, + "slug": "wal-recovery", + "filename": "issue-249-wal-recovery.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: Failure-safe checkpoint WAL restoration. Title: \"#249\" and \"WAL 恢复必须失败安全\". Status tag \"本分支新增代码 · 待收口\". Show a concrete staged file-flow diagram: old active log folder \"旧 WAL\" moves to a quarantined folder \"隔离\"; blue source files \"checkpoint tail\" copy to a staging folder \"staging\" then a publication gate labeled \"校验 / 发布\" into \"新 WAL\". At the transitions show three small amber warning labels \"移动失败\", \"复制失败\", \"发布失败\". A protective frame around checkpoint tail communicates preserving the recovery source. Desired invariant text under a clearly marked \"目标\" footer: \"旧日志不重放\" and \"恢复源不丢失\". Do not depict a completed fix or guaranteed atomicity. No claim this is a Topling native engine bug; it is shared recovery code introduced by this integration branch.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/249", + "revision_prompts": [ + "Edit this referenced WAL recovery infographic to remove distracting invented filenames and to label the entire diagram as proposed work. Preserve title #249, subtitle, overall flow, Chinese typography, warning labels, protected checkpoint tail source, and footer. Remove every individual filename from all folders and source card, leaving only clean unnamed small file icons. Make 新 WAL folder and its file icons blue, matching staging and checkpoint tail, to unambiguously show it receives checkpoint data rather than the gray old WAL. Replace the annotation 完整保留 待人工处理 under 隔离 with \"目标：隔离旧日志\". Add a small clear label \"目标流程\" above the central diagram. Footer goal checkmarks should become neutral outlined unfilled circles, since this is pending work. Do not alter any other text. Opaque background." + ] + }, + { + "issue": 248, + "slug": "restart-investigation", + "filename": "issue-248-restart-investigation.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: A bounded observation requiring investigation, not a claimed Topling data-loss diagnosis. Title: \"#248\" and \"重启后不可见：先确认根因\". Prominent amber tag \"Investigation\". Upper timeline labeled \"历史现象\": \"clear\" → \"v1 写入 / 读取成功\" → \"首次 Server 重启\" → \"v1 查询为空\". Lower comparison timeline: \"v2 新写入\" → \"再次 Server 重启\" → \"仍可读取\". Show a magnifying-glass investigative node with three dotted hypothesis branches labeled \"缓存\", \"事务 / 图 ID\", \"路由 / 持久化\"; no branch should be marked as the cause. Footer \"单次历史现象 · 根因待归属\". Visually distinguish query invisibility from physical data destruction; no shattered disks or deleted-file imagery. Mention only Server restart, not Store crash.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/248", + "revision_prompts": [ + "Edit the referenced infographic with one factual correction. Replace ONLY the small red annotation 数据存在但查询不可见 underneath v1 查询为空 with the exact text \"物理数据状态待核实\". The original wording falsely asserted physical data exists; this is unknown. Preserve ALL other text, title, layout, issue number, colors, iconography and dimensions exactly. Opaque background." + ] + }, + { + "issue": 212, + "slug": "cf-lifecycle", + "filename": "issue-212-cf-lifecycle.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: Historical CF/session lifecycle fixes versus currently unresolved native shutdown warnings. Title: \"#212\" and \"DB / CF 生命周期收口\". A left section labeled \"历史已修复\" shows \"释放 session\" → \"保留 CF 清表\" → \"旧 SHA 三轮通过\" with modest green checkmarks confined to this historical section. A right section labeled \"当前残余待归因\" contains an amber native warning card with exact text \"db not closed\", connected by dotted investigative arrows to nodes \"Session 引用\", \"CF 注册\", \"JNI 生命周期\". A clear DB cylinder containing CF partitions connects the two sections without suggesting the entire issue is solved. Footer \"旧 SHA 通过 ≠ 当前全部闭环\". No claim that a new JNI binary was released or that all shutdown problems are native-specific.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/212", + "revision_prompts": [] + }, + { + "issue": 213, + "slug": "jni-provenance", + "filename": "issue-213-jni-provenance.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: Traceable JNI delivery rather than an opaque binary. Title: \"#213\" and \"JNI 产物可追溯交付\". Show a current compact input card \"JNI SNAPSHOT\" with a small established fingerprint badge \"SHA-256\" and label \"候选可用\". Next to it, a larger clearly labeled \"目标交付链\" flows through five illustrated modules: \"源码 / 子模块\" → \"工具链 / 构建参数\" → \"JNI + 校验和\" → \"签名 / 来源证明\" → \"Maven 发行\". Use outlined unfilled completion indicators for this target chain so it is obviously pending work. A small connective warning between current artifact and source says \"来源链待补齐\". Footer \"候选可用 ≠ 正式发行闭环\". No invented build versions, timestamps, hashes, certified seals, or claims of completed provenance.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/213", + "revision_prompts": [] + }, + { + "issue": 252, + "slug": "benchmark-plan", + "filename": "issue-252-benchmark-plan.png", + "prompt": "Use case: infographic-diagram.\nAsset type: GitHub engineering issue illustration for HugeGraph ToplingDB, one image in a coherent series.\nPrimary request: Generate a polished, accurate explanatory raster infographic, not a decorative banner. Wide landscape 16:9 composition, high resolution, crisp readable Chinese typography with short English code identifiers. Neutral near-white background, dark charcoal type, blue for the TP integration path, neutral gray for standard or out-of-scope paths, amber for uncertainty, restrained red for confirmed mismatch. Flat technical editorial illustration with subtly dimensional database/file modules, clean connectors, generous whitespace and strong visual hierarchy. No characters, mascots, brand logos, invented dashboards, photographic clutter or watermarks. Use only the exact short labels supplied below, no extra paragraphs. Keep all text large enough to read in a GitHub issue. The image is explanatory and cannot assert an unresolved issue is fixed. Any desired state must be clearly labeled 目标, never 已完成. Avoid invented performance numbers or signs of actual data loss. Preserve the technical distinctions in the brief. Include the issue number as a small but legible top label. Opaque background.\n\nSubject: A fair reproducible benchmark plan with no invented performance outcome. Title: \"#252\" and \"同条件比较 RocksDB 与 Topling\". Prominent neutral status \"Benchmark 待执行\". At the top, shared controls labeled \"同一源码\", \"固定数据\", \"相同资源\", \"记录实际 JNI\" feed two equally sized peer lanes \"RocksDB\" and \"Topling\". Both lanes pass through the same workload cards labeled \"写入\", \"点读\", \"范围 / 邻接\", \"混合负载\". At the bottom show an empty comparison report grid with three outlined round markers \"第 1 轮\", \"第 2 轮\", \"第 3 轮\", no numbers, no bars implying a winner. Footer \"仅 Linux 实测 · 不预设性能收益\". Do not insert fake speedups, percentages, latency values or benchmark results.", + "source_url": "https://github.com/hugegraph/hugegraph/issues/252", + "revision_prompts": [] + } + ] +} diff --git a/docs/toplingdb/toplingdb-development.md b/docs/toplingdb/toplingdb-development.md new file mode 100644 index 0000000000..6429dbcbc4 --- /dev/null +++ b/docs/toplingdb/toplingdb-development.md @@ -0,0 +1,315 @@ +# ToplingDB Developer Guide + +This guide describes the Topling integration maintained in this repository. +Read the [quickstart](toplingdb-quickstart.md) first if you only need to run it. + +The runtime support target is native Linux x86_64 and Linux containers on +macOS (Docker Desktop or OrbStack). Native macOS execution, including Intel, +is deliberately outside the ToplingDB matrix; do not treat that CI job as a +Topling image or container failure. + +## Scope and Design Priorities + +The integration focuses on Topling-specific adaptation, bug fixes, feature +compatibility, usability, and maintainable engine selection. Keep HugeGraph +on the standard RocksDB API with Easy Migrate owning native integration; +do not add Java-side SidePluginRepo management or a parallel storage API. +Reuse shared launch helpers and deployment topologies instead of maintaining +separate copies per provider. + +Track provider-independent Server, PD, Store, schema/cache, client reconnect, +and deployment defects in their own issues or PRs. Mark them `ignore` in the +Topling task: this excludes them from its backlog and completion denominator, +not from product ownership, and does not mean they passed. Preserve reproduction +and dependency links. Such defects may limit an individual experiment; record +that limit and continue independent Topling checks rather than making all +pre-existing platform gaps prerequisites for this integration. + +Regressions introduced by this PR remain in scope even in shared code. +Unattributed native or WAL failures need a minimal diagnosis before exclusion. +Provider selection, native ownership, runtime packaging, data-directory safety, +and standard RocksDB compatibility remain integration requirements. + +Review engine switching for explicit configuration, actionable startup errors, +component-local JNI isolation, consistent defaults and versions, and recoverable +configuration changes. Do not imply live hot switching or automatic conversion +of an existing data directory. Use focused local tests and source/image/JNI-bound +Linux acceptance; performance experiments belong on the Linux test host. + +Track native CF lifecycle in [#212](https://github.com/hugegraph/hugegraph/issues/212), +release-grade JNI delivery in [#213](https://github.com/hugegraph/hugegraph/issues/213), +and current coordination in [#240](https://github.com/hugegraph/hugegraph/issues/240). +Issue [#214](https://github.com/hugegraph/hugegraph/issues/214) retains historical +integration and publication milestones. +Keep feature-merge criteria distinct from official-release requirements. +Current configuration-source and multi-graph isolation gaps are tracked in +[#250](https://github.com/hugegraph/hugegraph/issues/250), +[#251](https://github.com/hugegraph/hugegraph/issues/251), and +[#253](https://github.com/hugegraph/hugegraph/issues/253); adapter regression +coverage and diagnostic classification are tracked in +[#254](https://github.com/hugegraph/hugegraph/issues/254) and +[#255](https://github.com/hugegraph/hugegraph/issues/255). + +## Runtime Contract + +HugeGraph keeps the RocksDB Java API. Topling Easy Migrate supplies the native +runtime and configuration at process startup. + +```text +component config + -> preload-topling.sh validates rocksdb or topling + -> component-local JAR joins CLASSPATH + -> component-local native library joins LD_PRELOAD + -> Easy Migrate YAML is exported + -> existing RocksDB Java calls open and close DB/CF handles +``` + +The supported providers are exactly `rocksdb` and `topling`. Duplicate +configuration values must agree. Missing Topling files or dependencies stop +startup. PD and Store never search a neighboring Server distribution. + +## Source Layout + +| Area | Path | +|---|---| +| Shared helper source | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh` | +| Package-local preparation | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh` | +| Startup selection | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh` | +| Checked-in Topling JAR | `hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/` | +| Standalone Easy Migrate YAML | `hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml` | +| PD Easy Migrate YAML | `hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml` | +| Store Easy Migrate YAML | `hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml` | +| Topling distribution generator | `install-dist/scripts/build-topling-distribution.sh` | +| Provider data marker | `hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh` | +| Docker build graph | `docker/bake.hcl` | +| Focused shell tests | `hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-*.sh` | + +The PD and Store assembly descriptors copy the canonical helper scripts into +their own `bin/` directories. Change the canonical Server copy, then verify all +three assembled copies are identical. + +## Build Standard and Topling Distributions + +Use Linux x86_64. The distribution generator also requires `rsync`, `unzip`, +`tar`, and `sha256sum`. + +```bash +VERSION=$(mvn help:evaluate \ + -Dexpression=project.version -q -DforceStdout) + +mvn clean package \ + -pl hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-dist,hugegraph-store/hg-store-dist \ + -am -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -ntp + +for component in server pd store; do + install-dist/scripts/build-topling-distribution.sh \ + "$component" "$VERSION" +done +``` + +The standard distributions contain the selection helpers but no +`lib/topling/rocksdbjni*.jar` or prepared Topling native library. Selecting +Topling from a standard distribution must fail. + +When a Topling distribution is switched back to `rocksdb`, the Server and +`init-store` launchers exclude the canonical `lib/topling/` subtree (including +symlink aliases) from their classpaths; the optional Topling JAR is only added +after the startup selector accepts `topling`. + +The Topling distributions add only build-time runtime files. The generator +excludes PID files, logs, data directories, and prepared runtime state from the +standard distribution before creating the Topling tarball. It writes +`lib/topling/runtime.properties` with the component, project version, JAR name, +and JAR SHA-256. + +That SHA-256 identifies the local input. It is not an official release +signature or a full source-to-binary provenance record. Issue +[#213](https://github.com/hugegraph/hugegraph/issues/213) tracks immutable +coordinates, toolchain and CPU baseline, licensing, SBOM, signatures, and +consumer verification. + +## Provider Configuration + +Server reads `rocksdb.provider` from local RocksDB graph `.properties` files +in the `graphs` directory selected by the effective REST configuration. +Relative graph directories resolve from the Server distribution root, as in +the Java launchers. All local RocksDB graphs in one JVM must agree; a missing +provider selects standard RocksDB. Remote and in-memory graphs do not select JNI. + +For Server, `TOPLINGDB_ROCKSDB_PROVIDER` checks that selection; it cannot override +a different graph provider. A conflict stops startup before Java runs. +Docker still generates `hugegraph.properties` in that directory from +`HG_SERVER_ROCKSDB_PROVIDER`; the file must exist. Additional graphs must use +the same effective value. Direct launch does not require that primary filename. + +Example graph property: + +```properties +rocksdb.provider=topling +``` + +PD and Store read the provider under the root `rocksdb` YAML section: + +```yaml +rocksdb: + provider: topling +``` + +Keep only one effective value for a component. The loader rejects unknown or +conflicting values. The default is standard RocksDB when no provider is set. + +Server startup accepts plain properties and supported escaped path characters. +It rejects includes, continuations, escaped keys, duplicate relevant properties, +and interpolated/list values that cannot be safely matched to Java configuration. +Server startup also rejects non-empty `rocksdb.data_disks` until those optimized +roots can be enumerated safely. These errors name the affected property/file; +they do not select a fallback runtime. + +Every Topling functional test must use a readable Easy Migrate YAML. Removing +`TOPLINGDB_EASY_MIGRATE_CONF` reduces a test to Java ABI coverage and does not +exercise the Topling runtime. + +## Focused Tests + +Run the platform-independent selection and packaging tests first: + +```bash +TRAVIS_DIR=hugegraph-server/hugegraph-dist/src/assembly/travis + +"$TRAVIS_DIR/test-topling-runtime-selection.sh" +"$TRAVIS_DIR/test-topling-native-diagnostic.sh" +"$TRAVIS_DIR/test-topling-docker-entrypoints.sh" +"$TRAVIS_DIR/test-topling-runtime-packaging.sh" \ + "hugegraph-server/apache-hugegraph-server-$VERSION" \ + "hugegraph-pd/apache-hugegraph-pd-$VERSION" \ + "hugegraph-store/apache-hugegraph-store-$VERSION" +``` + +On Linux x86_64, validate each generated Topling distribution: + +```bash +"$TRAVIS_DIR/test-topling-distribution.sh" \ + server \ + "hugegraph-server/apache-hugegraph-server-$VERSION" \ + "hugegraph-server/apache-hugegraph-server-$VERSION-topling" + +"$TRAVIS_DIR/test-rocksdb-runtime.sh" \ + topling \ + "hugegraph-server/apache-hugegraph-server-$VERSION-topling" +``` + +Repeat the distribution and runtime checks for PD and Store. The CI jobs +`server-rocksdb-runtime` and `distributed-rocksdb-runtime` build a matrix across +standard RocksDB and ToplingDB. Standard runtime checks and both distribution +contracts are required. The synthetic Topling column-family lifecycle check is +allowed a non-blocking exception only after a separate runtime mapping/read/write/close +probe passes and the synthetic CF phase exits with SIGABRT and the exact known +assertion for issue #212. Missing configuration, class loading, mapping, IO and +unknown failures fail the job. Probe/lifecycle logs and JAR/native SHA-256 +identities are uploaded even on failure. Only the exact producer warning for an +unavailable optional DirectByteBuffer constructor is excluded from the error +scan; successful probe status and phase remain mandatory. Image-backed service lifecycle tests +remain required. The jobs also contaminate the standard build +with known runtime-state fixtures and verify that the Topling directory and +tarball remain clean. + +Run repository checks before pushing: + +```bash +mvn editorconfig:check -ntp +mvn clean compile -Dmaven.javadoc.skip=true -ntp +git diff --check +``` + +Use ShellCheck on each changed shell script. Existing warnings in untouched +lines do not justify unrelated cleanup in a Topling change. + +## Docker Images + +Server, PD, and Store Dockerfiles have explicit `standard` and `topling` +targets. Build the complete Topling deployment set with Bake: + +```bash +RUNTIME_VARIANT=topling \ +IMAGE_TAG=topling \ +docker buildx bake --file docker/bake.hcl +``` + +The Topling variant is Linux x86_64 only. It builds +`hugegraph/hugegraph`, `hugegraph/pd`, and `hugegraph/store` from their +`topling` targets. `hugegraph/server` remains the standard HStore Server and +receives the same tag as the compatible deployment set. + +An HStore Server image remains free of a local Topling JAR and native library. +PD and Store own their local runtime. + +Published candidates carry `org.opencontainers.image.source`, +`org.opencontainers.image.revision`, and +`org.apache.hugegraph.rocksdb-runtime` labels. Verify the exact source commit +before accepting a mutable deployment tag: + +```bash +docker image inspect hugegraph/hugegraph:topling \ + --format '{{json .Config.Labels}}' +``` + +Run the generic standalone, minimal HStore, or 3+3+3 Compose file with +provider parameters injected through the environment. Local development can +use `local/hugegraph-{pd,store}:topling` with +`HUGEGRAPH_{PD,STORE}_BUILD_TARGET=topling`; published candidates use +`hugegraph/{pd,store}:topling` and `HUGEGRAPH_{PD,STORE}_PULL_POLICY=always`. +Set `HUGEGRAPH_SERVER_IMAGE=hugegraph/server:topling` and +`HUGEGRAPH_SERVER_PULL_POLICY=always` only when the matching HStore image is +intended. Keep the provider-specific volume names and data roots explicit. +The repository deliberately maintains one Compose topology per deployment +shape rather than separate Topling files. + +Every local RocksDB owner uses a provider-specific data root. The Server entrypoint +validates `.hugegraph-rocksdb-provider` for the default root and every effective +local graph data/WAL path before initialization or Java startup. Marked ancestors +and configured paths are checked so a nested path cannot bypass isolation. +Primary configuration generation precedes enumeration; no database is opened +until validation succeeds. PD and Store validate their component data roots. Topling rejects unmarked non-empty data. Standard RocksDB accepts existing +non-empty legacy data without a marker; new or empty configured roots are +claimed atomically, including bare Server startup on macOS. Both reject +conflicting markers and storage symlinks. Topling/forced validation uses a +pinned-directory lock and rechecks emptiness after creating its temporary marker +so a competing child owner cannot be overwritten by a deployment-root marker. +On Linux, Server launchers also check each configured local graph's `m`, `g`, +and `s` database mount points before claiming graph roots or starting Java. +The Java recovery lock still guards direct database opens. + +## Native Runtime Changes + +When replacing `rocksdbjni*.jar`: + +1. Keep exactly one Topling JAR in the checked-in source directory. +2. Run `bin/prepare-topling.sh` in every generated component distribution. +3. Check `ldd library/librocksdbjni-linux64.so` for unresolved dependencies. +4. Run real DB and column-family create, close, reopen, restart, and persistence + tests with the component Easy Migrate YAML enabled. +5. Record the JAR SHA-256 and the exact source and toolchain evidence available. + +Issue [#212](https://github.com/hugegraph/hugegraph/issues/212) tracks native +DB/column-family lifecycle behavior. Real Server and 1+1+1 HStore tests must +cover CRUD, clean HugeGraph transaction close, container recreation, and +persistence. The current native runtime can still report a SidePluginRepo +bookkeeping warning after HugeGraph closes successfully. Keep that upstream +evidence separate from the provider-selection and data-isolation gates. ABI +loading or `ldd` output cannot establish service correctness. + +## Review Boundaries + +A complete Topling change must keep these properties: + +- standard artifacts have no Topling JAR or native library; +- Server, PD, and Store load only their component-local runtime; +- an HStore Server does not load Topling locally; +- standard and Topling data directories stay separate; +- invalid or incomplete Topling selection fails before service startup; +- shutdown uses the normal component lifecycle; +- documentation names the exact configuration file and verification command. + +Do not claim hot switching, automatic data migration, safe reuse of a +Topling-modified directory by standard RocksDB, or release provenance that the +artifact metadata does not prove. diff --git a/docs/toplingdb/toplingdb-hstore-integration.md b/docs/toplingdb/toplingdb-hstore-integration.md new file mode 100644 index 0000000000..be6b21cbce --- /dev/null +++ b/docs/toplingdb/toplingdb-hstore-integration.md @@ -0,0 +1,95 @@ +# ToplingDB and HStore Integration + +## Deployment Model + +ToplingDB and HStore solve different problems: + +- ToplingDB is an optional RocksDB-compatible native runtime. +- HStore is HugeGraph's distributed storage backend. + +In an HStore deployment, HugeGraph Server sends storage requests to Store. It +does not open a local RocksDB database. + +```text +HugeGraph Server + HStore client + | + v +PD cluster ---------------- Store cluster +local RocksDB metadata local RocksDB data +optional Topling runtime optional Topling runtime +``` + +ToplingDB therefore applies independently to PD and Store. Each process keeps +its existing RocksDB Java calls and loads its own native runtime and Easy +Migrate configuration. + +## Runtime Setup + +PD startup sets: + +```bash +TOPLINGDB_EASY_MIGRATE_CONF="$PD_HOME/conf/rocksdb_pd.yaml" +``` + +Store startup sets: + +```bash +TOPLINGDB_EASY_MIGRATE_CONF="$STORE_HOME/conf/rocksdb_store.yaml" +``` + +Both distributions carry the same `preload-topling.sh` helper and load only +their component-local Topling JAR and native library. PD and Store do not +discover or depend on a neighboring Server distribution. This is shared source +tooling, not a shared Java provider or shared DB owner. + +## Configuration + +Set the provider for each component that should use ToplingDB: + +```yaml +rocksdb: + provider: topling +``` + +PD and Store use distinct Easy Migrate YAML files and distinct HTTP ports. +Their `DBOptions.default` mappings remain the global fallbacks, while +`DBOptions.log` remains the specialized log-database profile. + +The Server's graph configuration does not enable a local ToplingDB instance +when the backend is HStore. + +## Open and Close Behavior + +PD and Store continue to use normal RocksDB Java APIs. The preloaded native hook +applies options and tracks DB/CF lifecycle. Neither component reflectively +creates a repository or calls an alternate `openDB`. + +Shutdown is also unchanged: + +```text +SIGTERM + -> PD/Store graceful shutdown + -> Java closes CF handles and RocksDB + -> native MaybeForgetCF and MaybeForgetDB + -> process exits +``` + +Store's existing shutdown timeout is a general process boundary. A timeout does +not justify calling `SidePluginRepo.closeAllDB()` and this integration does not +change the broader Store thread-exit behavior. + +## Validation + +Validate PD and Store separately: + +1. Confirm the component provider is `topling`. +2. Confirm the startup log reports the expected + `TOPLINGDB_EASY_MIGRATE_CONF`. +3. Confirm only the prepared component native library is mapped. +4. Run DB/CF create, reopen, drop, and recreate operations with Easy Migrate + enabled. +5. Stop the component with SIGTERM and verify normal process exit. + +An ABI-only check may inspect the JAR, Topling API marker, and native mapping +without opening a database. It is not a storage-functionality test. diff --git a/docs/toplingdb/toplingdb-operations.md b/docs/toplingdb/toplingdb-operations.md new file mode 100644 index 0000000000..5dd40eb889 --- /dev/null +++ b/docs/toplingdb/toplingdb-operations.md @@ -0,0 +1,105 @@ +# ToplingDB Operations Guide + +## Preflight + +Before startup: + +1. Confirm the component configuration selects `topling`. +2. Run the installation step for that component. +3. Confirm its Easy Migrate YAML and prepared native library are readable. +4. Check that the configured HTTP port is available and restricted to a + trusted interface. +5. Create and verify a complete pre-migration RocksDB checkpoint or backup. + +The expected configuration files are: + +| Component | File | +|---|---| +| Standalone Server | `conf/toplingdb.yaml` | +| PD | `conf/rocksdb_pd.yaml` | +| Store | `conf/rocksdb_store.yaml` | + +## Monitoring + +Monitor read/write latency, compaction backlog, cache usage, WAL growth, disk +space, native memory, and process health. The sample configurations keep the +Topling HTTP endpoint disabled by default (`auto_start_http: false`). If it is +needed, enable it explicitly; it can show native state but has no +authentication and must not be exposed directly to untrusted networks. + +Use the component logs to confirm the selected configuration path and native +library. Do not infer ToplingDB activation from the JAR name alone. + +## Tuning + +Change one YAML setting group at a time and benchmark with a representative +workload. Preserve: + +- `DBOptions.default` as the global fallback; +- the dedicated `DBOptions.log` profile; +- component-specific HTTP ports; +- a memory budget that includes JVM heap, block cache, memtables, native + allocations, and background jobs. + +## Graceful Stop and Restart + +Use the normal stop script or SIGTERM. Do not stop a separate “ToplingDB +process”; none exists. + +```text +SIGTERM + -> HugeGraph/PD/Store graceful shutdown + -> normal CF/DB close + -> native MaybeForgetCF / MaybeForgetDB + -> JVM exit +``` + +Do not invoke `SidePluginRepo.closeAllDB()`. Store's stop script already has a +bounded wait. If it reports a timeout, collect thread dumps and component logs +and investigate the ordinary Store shutdown path before any restart. + +## Upgrade and Rollback + +Drain traffic and stop the component cleanly before changing the runtime or +YAML. Validate JAR/native compatibility and the YAML schema in staging first. + +A provider switch is not a data rollback. If the upgraded Topling runtime has +written data and rollback is required, stop all writers and restore the full +pre-upgrade snapshot into an empty data directory with the matching previous +runtime. + +## Interrupted Standalone Snapshot Restore + +Standalone RocksDB snapshot restore keeps its checkpoint until the replacement +has been installed and reopened. A sibling `.resume-pending` file +records the source checkpoint and WAL location before data changes. Every new +HugeGraph open checks it before native recovery; if installation was interrupted, +it retries that checkpoint and replaces live WAL with verified checkpoint logs. +Missing sources, incomplete metadata or a changed WAL configuration stop opening. + +Preserve the checkpoint, pending marker and configured paths when diagnosing a +failure. Restore access/space and retry normal startup with the same runtime and +configuration. Do not delete the pending marker to bypass the guard: that can +allow stale or partial logs to replay. A successful native reopen clears the +pending marker and then attempts source cleanup, retaining historical +consume-on-success behavior. + +A sibling `.resume-lock` file serializes cooperating HugeGraph opens +and restores. The OS lock is held until the database closes; the lock file is +retained to avoid racing another opener. Its presence alone does not indicate +an active owner. Mount the parent data root (for example `rocksdb-data`), +not an individual store directory such as `data/g`. Opens reject a store directory +that is a separate volume mount; Linux also detects same-filesystem bind mounts +using the current process mount table. Aliases would hide the sibling guards, +and the existing directory-replacement restore cannot remove +a mount point. Server launchers preflight all local graph store directories +before Java starts. Mounts present during that check are rejected before sibling +stores initialize; keep the mount layout stable throughout startup. +Do not run older binaries or unrelated writers concurrently on the +same directories; they do not honor this recovery protocol. + +Independent WAL, WAL inside data, and data inside a WAL root use in-place log +replacement. Preserve WAL symlink configuration across retries. The local IO +fault tests cover interrupted operations and reopening; they do not establish +power-cut durability. This protocol is for standalone RocksDB adapter restore, +not HStore graph-level or multi-partition snapshots. diff --git a/docs/toplingdb/toplingdb-quickstart.md b/docs/toplingdb/toplingdb-quickstart.md new file mode 100644 index 0000000000..e01cb55f85 --- /dev/null +++ b/docs/toplingdb/toplingdb-quickstart.md @@ -0,0 +1,346 @@ +# ToplingDB Quickstart + +This guide covers a source-built ToplingDB distribution on Linux x86_64. On +macOS, run the published Linux image through Docker Desktop or OrbStack +container mode. Native macOS execution (including Intel) is outside this +ToplingDB support matrix. Use a standard HugeGraph distribution when you want +standard RocksDB. + +## Choose the Correct Component + +| What you run | Topling package | Provider configuration | +|---|---|---| +| Standalone Server | `apache-hugegraph-server--topling` | `conf/graphs/hugegraph.properties` | +| PD | `apache-hugegraph-pd--topling` | `conf/application.yml` | +| Store | `apache-hugegraph-store--topling` | `conf/application-pd.yml` | +| HStore Server | Standard HStore Server | No local Topling setting | + +PD and Store select their providers independently. An HStore-backed Server does +not load a Topling JAR or native library. + +Set standalone data with `rocksdb.data_path` in +`conf/graphs/hugegraph.properties`, PD metadata with `pd.data-path` in +`conf/application.yml`, and Store data with `app.data-path` in +`conf/application-pd.yml`. + +## Prerequisites + +Topling distributions and images currently target Linux x86_64. The published +images use the `linux/amd64` platform, so macOS container mode may need +`--platform linux/amd64`; the native packages required by the bundled library +are installed in the images. The preparation script reports any unresolved +dependency and stops. + +## Docker Images + +The `topling` image tag names one compatible deployment set: + +| Image | Role | +|---|---| +| `hugegraph/hugegraph:topling` | Standalone Server with local ToplingDB | +| `hugegraph/pd:topling` | PD metadata on ToplingDB | +| `hugegraph/store:topling` | Store data on ToplingDB | +| `hugegraph/server:topling` | HStore Server without a local Topling runtime | + +Run the standalone image with its provider-specific data volume: + +```bash +docker volume create hugegraph-topling-data +docker run -d \ + --pull always \ + --platform linux/amd64 \ + --name hugegraph-topling \ + -p 8080:8080 \ + -v hugegraph-topling-data:/hugegraph-server/topling-data \ + hugegraph/hugegraph:topling + +curl --fail http://127.0.0.1:8080/versions +``` + +The same standalone image can be run with the generic repository Compose file. +Inject the provider and volume parameters so the standard volume cannot be +reused: + +```bash +export HUGEGRAPH_SERVER_IMAGE='hugegraph/hugegraph:topling' +export HUGEGRAPH_SERVER_PULL_POLICY='always' +export HG_SERVER_ROCKSDB_PROVIDER='topling' +export HG_SERVER_DATA_PATH='/hugegraph-server/topling-data' +export HG_SERVER_ENFORCE_PROVIDER_MARKER='true' +export HUGEGRAPH_SERVER_VOLUME='server-topling-data' +docker compose -f docker/docker-compose.yml \ + up -d --wait +``` + +Run a source-checkout 1+1+1 HStore stack with Topling PD and Store images. +The generic Compose files accept the same parameters for local builds or +published images; the HStore Server itself remains standard: + +```bash +export HUGEGRAPH_ADMIN_PASSWORD='replace-with-a-strong-password' +export HG_PD_AUTH_SECRET_KEY='replace-with-a-separate-strong-pd-secret' +export HUGEGRAPH_PD_IMAGE='hugegraph/pd:topling' +export HUGEGRAPH_PD_PULL_POLICY='always' +export HUGEGRAPH_PD_VOLUME='pd-topling-data' +export HG_PD_ROCKSDB_PROVIDER='topling' +export HG_PD_DATA_PATH='/hugegraph-pd/topling-pd-data' +export HG_PD_ENFORCE_PROVIDER_MARKER='true' +export HUGEGRAPH_STORE_IMAGE='hugegraph/store:topling' +export HUGEGRAPH_STORE_PULL_POLICY='always' +export HUGEGRAPH_STORE_VOLUME='store-topling-data' +export HG_STORE_ROCKSDB_PROVIDER='topling' +export HG_STORE_DATA_PATH='/hugegraph-store/topling-storage' +export HG_STORE_ENFORCE_PROVIDER_MARKER='true' +export HUGEGRAPH_SERVER_IMAGE='hugegraph/server:topling' +export HUGEGRAPH_SERVER_PULL_POLICY='always' + +docker compose \ + -f docker/docker-compose-hstore.yml \ + up -d --wait pd store server +``` + +For source-built PD and Store images, use `local/hugegraph-{pd,store}:topling`, +set both pull policies to `never`, add `HUGEGRAPH_{PD,STORE}_BUILD_TARGET=topling`, +and append `-f docker/docker-compose.dev.yml`. The generic files mount the +separate `pd-topling-data` and `store-topling-data` volumes. The HStore Server +does not load a local Topling library. + +For a published 3+3+3 stack, select all three deployment images explicitly: + +```bash +export HUGEGRAPH_ADMIN_PASSWORD='replace-with-a-strong-password' +export HG_PD_AUTH_SECRET_KEY='replace-with-a-separate-strong-pd-secret' +export HUGEGRAPH_PD_IMAGE=hugegraph/pd:topling +export HUGEGRAPH_PD_PULL_POLICY=always +export HG_PD_ROCKSDB_PROVIDER=topling +export HG_PD_DATA_PATH=/hugegraph-pd/topling-pd-data +export HG_PD_ENFORCE_PROVIDER_MARKER=true +export HUGEGRAPH_PD0_VOLUME=hg-pd0-topling-data +export HUGEGRAPH_PD1_VOLUME=hg-pd1-topling-data +export HUGEGRAPH_PD2_VOLUME=hg-pd2-topling-data +export HUGEGRAPH_STORE_IMAGE=hugegraph/store:topling +export HUGEGRAPH_STORE_PULL_POLICY=always +export HG_STORE_ROCKSDB_PROVIDER=topling +export HG_STORE_DATA_PATH=/hugegraph-store/topling-storage +export HG_STORE_ENFORCE_PROVIDER_MARKER=true +export HUGEGRAPH_STORE0_VOLUME=hg-store0-topling-data +export HUGEGRAPH_STORE1_VOLUME=hg-store1-topling-data +export HUGEGRAPH_STORE2_VOLUME=hg-store2-topling-data +export HUGEGRAPH_SERVER_IMAGE=hugegraph/server:topling +export HUGEGRAPH_SERVER_PULL_POLICY=always + +docker compose -f docker/docker-compose-3pd-3store-3server.yml \ + up -d --wait +``` + +Each PD and Store instance receives a distinct Topling volume. For local Bake +images, use the same variables with the tags produced by `docker/bake.hcl`. +No Topling-specific Compose file is required or maintained. + +The `topling` tag is mutable. The commands above force a pull so a cached image +cannot silently stand in for the current deployment set. Pin the registry +digest when an exact build must be reproduced, and verify the image source, +revision, and runtime labels during acceptance. + +## Build Distributions from Source + +Build on Linux x86_64 with Java 11+, Maven 3.5+, `rsync`, `unzip`, and `tar`. +Running a Linux Server distribution with local RocksDB also requires +util-linux 2.37+ `mountpoint` on `PATH` for the database mount preflight. + +```bash +git clone https://github.com/hugegraph/hugegraph.git +cd hugegraph +git switch toplingdb + +VERSION=$(mvn help:evaluate \ + -Dexpression=project.version -q -DforceStdout) + +mvn clean package \ + -pl hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-dist,hugegraph-store/hg-store-dist \ + -am -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -ntp +``` + +The build creates standard distributions first. Generate only the Topling +distributions you need: + +```bash +install-dist/scripts/build-topling-distribution.sh server "$VERSION" +install-dist/scripts/build-topling-distribution.sh pd "$VERSION" +install-dist/scripts/build-topling-distribution.sh store "$VERSION" +``` + +Each command creates a directory and a matching `.tar.gz` file beside the +standard distribution. The Topling package contains: + +```text +bin/prepare-topling.sh +bin/preload-topling.sh +lib/topling/rocksdbjni*.jar +lib/topling/runtime.properties +library/librocksdbjni-linux64.so +``` + +The generator selects `provider=topling` and prepares the native runtime. Run +`bin/prepare-topling.sh` again after replacing the bundled Topling JAR. + +## Provider Data Markers + +Docker entrypoints validate `.hugegraph-rocksdb-provider` before starting a +JVM. Server checks the primary root plus data and WAL paths of all local +RocksDB graphs in the directory selected by `graphs` in REST configuration. +Additional graph storage roots must be mounted or created before startup; +matching marked ancestors can own their descendant paths. All local graph +providers must agree. The marker records the component and provider. A mismatched +marker always stops startup. Topling also rejects an unmarked, non-empty data directory. +The configured data root must already exist as a real directory; mount or +create it before startup. Symlinked path components are rejected. + +Standard RocksDB accepts existing non-empty unmarked data for backward +compatibility. New or empty configured directories receive a standard marker +through an atomic claim; bare standard startup can create missing directories. +New deployments should still use the image defaults: + +| Component | Standard data root | Topling data root | +|---|---|---| +| Server | `/hugegraph-server/rocksdb-data` | `/hugegraph-server/topling-data` | +| PD | `/hugegraph-pd/pd_data` | `/hugegraph-pd/topling-pd-data` | +| Store | `/hugegraph-store/storage` | `/hugegraph-store/topling-storage` | + +The marker prevents accidental reuse. It does not convert data between +providers. + +## Standalone Server + +Enter the generated Server distribution: + +```bash +cd "hugegraph-server/apache-hugegraph-server-$VERSION-topling" +``` + +Confirm the provider and assign a data directory that no standard RocksDB +process uses. For Server, `TOPLINGDB_ROCKSDB_PROVIDER` must agree with graph +configuration; change `rocksdb.provider` explicitly rather than using that +variable to override it. Custom graph directories use the REST `graphs` +property and resolve relative to the Server distribution root: + +```properties +# conf/graphs/hugegraph.properties +backend=rocksdb +rocksdb.provider=topling +rocksdb.data_path=/srv/hugegraph/topling/server +``` + +Initialize and start the Server: + +```bash +bin/init-store.sh +bin/start-hugegraph.sh +curl --fail http://127.0.0.1:8080/versions +``` + +The startup output must include the selected Easy Migrate configuration: + +```text +[preload-topling] TOPLINGDB_EASY_MIGRATE_CONF=.../conf/toplingdb.yaml +``` + +Stop with the normal HugeGraph script: + +```bash +bin/stop-hugegraph.sh +``` + +Restart the same Topling distribution and read previously written data before +you accept persistence for that deployment. + +## Distributed HStore + +Build one Topling distribution for PD and one for Store. Keep the HStore Server +on its normal HStore distribution. + +In every PD distribution, enable ToplingDB and use a provider-specific metadata +directory: + +```yaml +# conf/application.yml +rocksdb: + provider: topling + option-path: ./conf/rocksdb_pd.yaml + +pd: + data-path: /srv/hugegraph/topling/pd +``` + +In every Store distribution, enable ToplingDB: + +```yaml +# conf/application-pd.yml +rocksdb: + provider: topling +``` + +Set a provider-specific Store data directory in the Store service +configuration: + +```yaml +# conf/application-pd.yml +app: + data-path: /srv/hugegraph/topling/store +``` + +Configure the normal PD, Store, and HStore network addresses as described in +the [distributed deployment guide](../../hugegraph-store/docs/deployment-guide.md). +Start PD before Store, then start the HStore-backed Server: + +```bash +# Run from the corresponding distribution directory +bin/start-hugegraph-pd.sh +bin/start-hugegraph-store.sh +bin/start-hugegraph.sh +``` + +Check each PD and Store log for its own Easy Migrate path. PD must use +`conf/rocksdb_pd.yaml`; Store must use `conf/rocksdb_store.yaml`. The HStore +Server must not report a local Topling runtime. + +## Topling HTTP Monitor + +The sample Easy Migrate files bind their HTTP monitors to loopback, but set +`auto_start_http: false`, so the monitor is disabled by default: + +| Component | Configuration | Default address | +|---|---|---| +| Server | `conf/toplingdb.yaml` | `127.0.0.1:2011` | +| PD | `conf/rocksdb_pd.yaml` | `127.0.0.1:2012` | +| Store | `conf/rocksdb_store.yaml` | `127.0.0.1:2013` | + +The endpoint has no authentication. Keep the loopback binding. Enable it only +when you need it: + +```yaml +http: + auto_start_http: true +``` + +Leave `auto_start_http: false` to keep the monitor disabled. + +## Return to Standard RocksDB + +A provider change is not a data conversion. Use this procedure: + +1. Stop all writers and stop the component cleanly. +2. Keep the Topling data directory unchanged. +3. Restore a full pre-Topling snapshot into a new, empty standard RocksDB data + directory. +4. Start the standard distribution with `provider=rocksdb`. +5. Validate schema, reads, writes, restart, and persistence before serving + traffic. + +Do not point standard RocksDB at a directory that ToplingDB has modified. + +## Startup Failures + +Startup stops when the provider is invalid or the component-local JAR, native +library, Easy Migrate file, or system dependency is missing. Follow the exact +error and see the [troubleshooting guide](toplingdb-troubleshooting.md). diff --git a/docs/toplingdb/toplingdb-security.md b/docs/toplingdb/toplingdb-security.md new file mode 100644 index 0000000000..1f1811ab80 --- /dev/null +++ b/docs/toplingdb/toplingdb-security.md @@ -0,0 +1,90 @@ +# ToplingDB Security Hardening Guide + +This document provides best practices for securing a ToplingDB deployment. It +covers file permissions, network access control, firewall rules, and additional +hardening measures that reduce the production attack surface. + +--- + +## 1. File Permissions + +Keep installation files owned by a deployment account. Give the service +account write access only to data, log, and runtime-state directories: + +```bash +chown root:hugegraph "$HUGEGRAPH_HOME"/bin/*.sh +chmod 750 "$HUGEGRAPH_HOME"/bin/*.sh + +for config in toplingdb.yaml rocksdb_pd.yaml rocksdb_store.yaml; do + test -e "$HUGEGRAPH_HOME/conf/$config" || continue + chown root:hugegraph "$HUGEGRAPH_HOME/conf/$config" + chmod 640 "$HUGEGRAPH_HOME/conf/$config" +done + +install -d -o hugegraph -g hugegraph -m 750 \ + "$HUGEGRAPH_HOME"/logs \ + /srv/hugegraph/topling/data \ + /run/hugegraph +``` + +- Replace `root` with the actual deployment owner when packages are installed + by a non-root account. +- The service account can read scripts and configuration but cannot modify the + startup chain, JARs, native libraries, or Easy Migrate configuration. +- Adjust the data path for Server, PD, or Store. Do not recursively change + ownership of the whole installation directory. + +--- + +## 2. Network Access Control + +Restrict network exposure by binding services to localhost or specific +interfaces: + +```yaml +# Localhost-only access +http: + listening_ports: '127.0.0.1:2011' +``` + +- Avoid binding to `0.0.0.0` unless absolutely necessary. +- Use reverse proxies (e.g., Nginx) or VPN tunnels if remote access is required. + +--- + +## 3. Firewall Rules + +The Topling HTTP monitor is disabled by default. If it is explicitly enabled, +insert a dedicated chain before broader `ACCEPT` rules. Preserve established +connections and define equivalent IPv4 and IPv6 policies: + +```bash +iptables -N HG_TOPLING 2>/dev/null || true +iptables -F HG_TOPLING +iptables -A HG_TOPLING -s 192.168.1.0/24 -j ACCEPT +iptables -A HG_TOPLING -j DROP +iptables -C INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT || + iptables -I INPUT 1 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +iptables -C INPUT -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING || + iptables -I INPUT 2 -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING + +ip6tables -N HG_TOPLING6 2>/dev/null || true +ip6tables -F HG_TOPLING6 +ip6tables -A HG_TOPLING6 -s 2001:db8:1234::/48 -j ACCEPT +ip6tables -A HG_TOPLING6 -j DROP +ip6tables -C INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT || + ip6tables -I INPUT 1 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +ip6tables -C INPUT -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING6 || + ip6tables -I INPUT 2 -p tcp -m multiport --dports 2011,2012,2013 \ + -j HG_TOPLING6 +``` + +- Replace both documentation prefixes with trusted production networks. +- Existing established connections remain allowed. Revoke them separately + through the host connection-tracking policy when immediate eviction is + required. +- Inspect the resulting rule order before applying it to a remote host. +- Consider using `firewalld` or `ufw` for simplified management. diff --git a/docs/toplingdb/toplingdb-troubleshooting.md b/docs/toplingdb/toplingdb-troubleshooting.md new file mode 100644 index 0000000000..644cf80849 --- /dev/null +++ b/docs/toplingdb/toplingdb-troubleshooting.md @@ -0,0 +1,104 @@ +# ToplingDB Troubleshooting + +## Startup Rejects the Easy Migrate Configuration + +Confirm that the startup log prints the expected component path: + +```text +[preload-topling] TOPLINGDB_EASY_MIGRATE_CONF=/path/to/config.yaml +``` + +Then check: + +- the file exists and is readable by the service user; +- the YAML uses valid indentation and supported Topling option names; +- `DBOptions.default` exists as the fallback; +- the dedicated `DBOptions.log` mapping is still present. + +Do not diagnose configuration loading by calling `SidePluginRepo` from Java. + +## Wrong or Missing Native Runtime + +Verify the prepared library and the process mapping: + +```bash +test -r /path/to/component/library/librocksdbjni-linux64.so +grep librocksdbjni /proc/$PID/maps +``` + +Only the selected component library should be mapped. A Topling class marker in +the JAR is useful for ABI diagnosis, but it does not prove that Easy Migrate +options were applied or that DB/CF operations work. + +## HTTP Port Conflict + +Check the component-specific `http.listening_ports` value and whether another +process already owns it: + +```bash +lsof -i : +``` + +The provided configurations use 2011 for Server, 2012 for PD, and 2013 for +Store. Confirm the actual value in the selected YAML. Components on one host +must not share a port. Keep the endpoint on a trusted interface because it has +no built-in authentication. + +## Database Lock Failure + +A lock error normally means another process owns the database or the service +user cannot access the data directory. + +1. Confirm there is only one owner for the database path. +2. Confirm directory permissions. +3. Confirm the component uses its own Easy Migrate configuration. +4. Inspect the component and RocksDB logs. + +Do not bypass the normal RocksDB open path and do not attempt a reflected +`openDB`. + +## Shutdown Timeout + +The expected lifecycle is: + +```text +SIGTERM -> graceful shutdown -> CF/DB close + -> native MaybeForgetCF/MaybeForgetDB -> exit +``` + +If Store exceeds its existing timeout: + +1. capture thread dumps before forcing termination; +2. inspect Store shutdown and background-thread logs; +3. confirm CF handles and RocksDB reached their normal close paths; +4. treat the result as a general Store lifecycle issue. + +Do not call `SidePluginRepo.closeAllDB()`. This integration does not provide a +second Java-side owner and does not expand the scope of general thread-exit +repairs. + +## DB/CF Test Gives a False Positive + +A ToplingDB test that unsets `TOPLINGDB_EASY_MIGRATE_CONF` before opening a +database proves only RocksDB Java ABI compatibility. It does not test ToplingDB +configuration or native lifecycle hooks. + +For a functional test, keep a readable component configuration in the +environment and exercise create, write, close, reopen, drop, and recreate +operations. Reserve no-configuration checks for ABI diagnostics that do not +open a database. + +## Server Provider or Graph Directory Conflict + +For Server, graph configuration selects the runtime. A conflicting +`TOPLINGDB_ROCKSDB_PROVIDER` stops startup; change the graph provider explicitly +or remove the conflicting variable. All local RocksDB graphs in the effective +REST `graphs` directory must agree, including graphs without an explicit +provider (which default to standard RocksDB). + +Use an existing graph directory and pre-created data roots. The launcher fails +on configuration syntax it cannot safely match to Java, rather than guessing a +provider or directory. Server startup marker validation rejects storage symlinks, +conflicting marked ancestors and unsupported optimized-disk layouts. Check the +reported file, property and path before retrying; do not delete a conflicting +provider marker to make startup pass. diff --git a/docs/toplingdb/toplingdb.md b/docs/toplingdb/toplingdb.md new file mode 100644 index 0000000000..77e9904a0e --- /dev/null +++ b/docs/toplingdb/toplingdb.md @@ -0,0 +1,134 @@ +# ToplingDB Support and Configuration + +ToplingDB is an optional RocksDB-compatible runtime for HugeGraph. The +integration uses ToplingDB Easy Migrate: HugeGraph keeps its existing RocksDB +Java API, while a native hook applies ToplingDB configuration and tracks the +DB/column-family lifecycle. + +New users should start with the [ToplingDB quickstart](toplingdb-quickstart.md). +Contributors should use the +[ToplingDB developer guide](toplingdb-development.md). + +## Architecture + +```text +component configuration + -> component-local prepare-topling.sh prepares the runtime + -> start script selects rocksdb or topling + -> preload-topling.sh exports TOPLINGDB_EASY_MIGRATE_CONF + -> JVM uses the existing RocksDB Java API + -> native Easy Migrate hook applies options and tracks DB/CF close +``` + +There is no HugeGraph Java provider SPI, reflected `SidePluginRepo`, or +separate ToplingDB service process. + +## Component Scope + +| Component | Local RocksDB owner | Easy Migrate YAML | +|---|---:|---| +| Standalone HugeGraph Server | Yes | `conf/toplingdb.yaml` | +| HugeGraph PD | Yes | `conf/rocksdb_pd.yaml` | +| HugeGraph Store | Yes | `conf/rocksdb_store.yaml` | +| HugeGraph Server using HStore | No | Not applicable | + +An HStore-backed Server is a remote client. Configure ToplingDB on PD and Store, +not on that Server process. + +## Enable ToplingDB + +ToplingDB is opt-in. Use the Topling distribution or image for the component, +set the provider in its configuration, then prepare the bundled runtime before +starting the service. + +For Server: + +```properties +rocksdb.provider=topling +``` + +For PD or Store, set the corresponding `rocksdb.provider` value in its +application YAML. + +Each Server, PD, and Store distribution carries the same runtime helpers. A +Topling distribution additionally carries its component-local JAR under +`lib/topling/`. Prepare that runtime without a source checkout: + +```bash +bin/prepare-topling.sh +``` + +The standard distribution carries the helpers but no Topling JAR or native +library. Selecting `topling` from a standard distribution therefore fails +explicitly instead of falling back to RocksDB. + +The normal start script sources its own `bin/preload-topling.sh`. For ToplingDB +it validates the prepared files and exports: + +```bash +TOPLINGDB_EASY_MIGRATE_CONF=/absolute/path/to/component/config.yaml +LD_LIBRARY_PATH=/path/to/component/library:... +LD_PRELOAD=.../librocksdbjni-linux64.so +``` + +A missing JAR, native library, or readable configuration is a startup error. +HugeGraph does not silently change providers. + +## Easy Migrate Configuration + +The component YAML defines ToplingDB options and optional HTTP observability. +The provided configurations use: + +- `DBOptions.default` as the global fallback; +- `DBOptions.log` as a dedicated profile for the log database. + +Keep both mappings. The log profile is intentional and must not be folded into +the fallback. + +The embedded HTTP endpoint has no authentication. Keep it bound to a trusted +interface and restrict network access. + +## DB and Column-Family Lifecycle + +HugeGraph continues to call `RocksDB.open()`, column-family APIs, handle +`close()`, and `RocksDB.close()`. Easy Migrate observes these calls in native +code, applies the matching configuration, retains live DB/CF state, and releases +it through `MaybeForgetCF` and `MaybeForgetDB` on normal close. + +Do not create or close a `SidePluginRepo` from HugeGraph Java code. + +## Shutdown + +Use the normal component stop script or send SIGTERM: + +```text +SIGTERM + -> component graceful shutdown + -> normal CF/DB close + -> native MaybeForgetCF / MaybeForgetDB + -> JVM exit +``` + +Store has an existing bounded shutdown wait. If it times out, diagnose the +ordinary Store thread/lifecycle issue. Do not call a Topling-specific +`closeAllDB()` workaround. + +## Compatibility and Rollback + +ToplingDB-specific options may produce WAL, SST, or metadata that standard +RocksDB cannot safely reopen. Switching only `rocksdb.provider` or replacing a +JAR is not a rollback. + +Before enabling ToplingDB, create a complete RocksDB-consistent checkpoint or +backup. To roll back, stop writers and restore the full pre-migration snapshot +with the matching standard RocksDB runtime. + +## Related Guides + +- [Documentation index](README.md) +- [Quickstart](toplingdb-quickstart.md) +- [Developer guide](toplingdb-development.md) +- [ToplingDB and HStore integration](toplingdb-hstore-integration.md) +- [Operations guide](toplingdb-operations.md) +- [Troubleshooting](toplingdb-troubleshooting.md) +- [Security guide](toplingdb-security.md) diff --git a/hugegraph-cluster-test/README.md b/hugegraph-cluster-test/README.md new file mode 100644 index 0000000000..adafd07982 --- /dev/null +++ b/hugegraph-cluster-test/README.md @@ -0,0 +1,22 @@ +# Cluster integration tests + +The simple and multi cluster suites start isolated PD, Store and Server processes +using the locally packaged distributions. Build the distributions first, then run +`mvn test -pl hugegraph-cluster-test/hugegraph-clustertest-test -am -P simple-cluster-test` +or the `multi-cluster-test` profile. + +Each node has a five-minute startup deadline. An exited process fails immediately, +even if an old log contains a ready marker. Failure messages identify the node and +its startup log. Each launch archives an existing startup log beside the new +log under a unique `.previous-*.log` name so readiness only uses the current attempt. A startup failure stops the task's node processes and preserves +their directories for diagnosis; normal teardown removes test directories. +An interrupted startup wait preserves the thread interrupt status. Shutdown waits +up to 20 seconds for a graceful exit and then up to 10 seconds after force-kill. +Interruptions do not reset these deadlines or abandon the wait; the interrupt +status is restored afterward. A process surviving force-kill fails teardown, and +its directory is retained. Cleanup still attempts the remaining nodes. These checks bound +startup diagnostics; the suites must still pass their functional assertions. + +The generated Server metadata `pd.peers` uses the same randomized PD addresses +as graph storage. Store readiness waits for the completed application startup +marker, rather than Spring's initial starting message. diff --git a/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template b/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template index 43979ec120..5b7c770641 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template +++ b/hugegraph-cluster-test/hugegraph-clustertest-dist/src/assembly/static/conf/rest-server.properties.template @@ -19,10 +19,13 @@ # could use '0.0.0.0' or specified (real)IP to expose external network access restserver.url=http://$REST_SERVER_ADDRESS$ # gremlin server url, need to be consistent with host and port in gremlin-server.yaml -#gremlinserver.url=http://$REST_SERVER_ADDRESS$ +gremlinserver.url=http://$GREMLIN_SERVER_ADDRESS$ graphs=./conf/graphs +# Server metadata and graph storage must use the same test PD cluster. +pd.peers=$PD_PEERS_LIST$ + # The maximum thread ratio for batch writing, only take effect if the batch.max_write_threads is 0 batch.max_write_ratio=80 batch.max_write_threads=0 diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml index b59648304f..b261e0f5b1 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/pom.xml @@ -51,7 +51,8 @@ org.slf4j slf4j-api - 2.0.9 + + 1.7.25 compile diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java index c71e4b07e1..6e1e9881f0 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ClusterConfig.java @@ -59,6 +59,7 @@ public ClusterConfig(int pdCnt, int storeCnt, int serverCnt) { for (int i = 0; i < serverCnt; i++) { ServerConfig serverConfig = new ServerConfig(); + serverConfig.setPDPeersList(pdGrpcList); serverConfigs.add(serverConfig); GraphConfig graphConfig = new GraphConfig(); graphConfig.setPDPeersList(pdGrpcList); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java index 569a11dddf..bffc775a0c 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/config/ServerConfig.java @@ -24,6 +24,7 @@ import static org.apache.hugegraph.ct.base.EnvUtil.getAvailablePort; import java.nio.file.Paths; +import java.util.List; import lombok.Getter; @@ -32,16 +33,23 @@ public class ServerConfig extends AbstractConfig { private final int rpcPort; private final int restPort; + private final int gremlinPort; public ServerConfig() { readTemplate(Paths.get(CONFIG_FILE_PATH + SERVER_TEMPLATE_FILE)); this.fileName = SERVER_PROPERTIES; this.rpcPort = getAvailablePort(); this.restPort = getAvailablePort(); + this.gremlinPort = getAvailablePort(); properties.put("REST_SERVER_ADDRESS", LOCALHOST + ":" + this.restPort); + properties.put("GREMLIN_SERVER_ADDRESS", LOCALHOST + ":" + this.gremlinPort); properties.put("RPC_PORT", String.valueOf(this.rpcPort)); } + public void setPDPeersList(List pdPeers) { + setProperty("PD_PEERS_LIST", String.join(",", pdPeers)); + } + public void setServerID(String serverID) { setProperty("SERVER_ID", serverID); } @@ -51,4 +59,3 @@ public void setRole(String role) { } } - diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java index 0c24860929..2e1aa07bba 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.hugegraph.ct.base.HGTestLogger; import org.apache.hugegraph.ct.config.ClusterConfig; @@ -28,6 +29,8 @@ import org.apache.hugegraph.ct.config.PDConfig; import org.apache.hugegraph.ct.config.ServerConfig; import org.apache.hugegraph.ct.config.StoreConfig; +import org.apache.hugegraph.ct.node.AbstractNodeWrapper; +import org.apache.hugegraph.ct.node.BaseNodeWrapper; import org.apache.hugegraph.ct.node.PDNodeWrapper; import org.apache.hugegraph.ct.node.ServerNodeWrapper; import org.apache.hugegraph.ct.node.StoreNodeWrapper; @@ -41,6 +44,8 @@ public abstract class AbstractEnv implements BaseEnv { private static final Logger LOG = HGTestLogger.ENV_LOG; + private boolean startupFailed; + protected ClusterConfig clusterConfig; protected List pdNodeWrappers; protected List serverNodeWrappers; @@ -71,9 +76,11 @@ protected void init(int pdCnt, int storeCnt, int serverCnt) { } for (int i = 0; i < serverCnt; i++) { - ServerNodeWrapper serverNodeWrapper = new ServerNodeWrapper(cluster_id, i); - serverNodeWrappers.add(serverNodeWrapper); ServerConfig serverConfig = clusterConfig.getServerConfig(i); + int gremlinPort = serverConfig.getGremlinPort(); + ServerNodeWrapper serverNodeWrapper = new ServerNodeWrapper(cluster_id, i, + gremlinPort); + serverNodeWrappers.add(serverNodeWrapper); serverConfig.setServerID(serverNodeWrapper.getID()); GraphConfig graphConfig = clusterConfig.getGraphConfig(i); if (i == 0) { @@ -87,47 +94,94 @@ protected void init(int pdCnt, int storeCnt, int serverCnt) { } public void startCluster() { - for (PDNodeWrapper pdNodeWrapper : pdNodeWrappers) { - pdNodeWrapper.start(); - while (!pdNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + try { + for (PDNodeWrapper node : this.pdNodeWrappers) { + startNode(node); } - } - for (StoreNodeWrapper storeNodeWrapper : storeNodeWrappers) { - storeNodeWrapper.start(); - while (!storeNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + for (StoreNodeWrapper node : this.storeNodeWrappers) { + startNode(node); + } + for (ServerNodeWrapper node : this.serverNodeWrappers) { + startNode(node); } + } catch (RuntimeException | Error failure) { + this.startupFailed = true; + try { + this.stopCluster(); + } catch (RuntimeException | Error cleanup) { + failure.addSuppressed(cleanup); + } + throw failure; } - for (ServerNodeWrapper serverNodeWrapper : serverNodeWrappers) { - serverNodeWrapper.start(); - while (!serverNodeWrapper.isStarted()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); + } + + private static void startNode(AbstractNodeWrapper node) { + node.start(); + awaitStarted(node, TimeUnit.MINUTES.toMillis(5)); + } + + static void awaitStarted(BaseNodeWrapper node, long timeoutMillis) { + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + try { + while (true) { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + if (!node.isAlive()) { + throw startupFailure(node, "process exited before becoming ready"); + } + if (node.isStarted()) { + return; + } + long remaining = deadline - System.nanoTime(); + if (remaining <= 0L) { + throw startupFailure(node, "startup timed out after " + timeoutMillis + " ms"); } + TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.SECONDS.toNanos(1))); } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while starting " + node.getID() + + "; log: " + node.getLogPath(), e); } } + private static IllegalStateException startupFailure(BaseNodeWrapper node, String reason) { + return new IllegalStateException("Node " + node.getID() + " " + reason + + "; log: " + node.getLogPath()); + } + public void stopCluster() { - for (ServerNodeWrapper serverNodeWrapper : serverNodeWrappers) { - serverNodeWrapper.stop(); - } - for (StoreNodeWrapper storeNodeWrapper : storeNodeWrappers) { - storeNodeWrapper.stop(); + List nodes = new ArrayList<>(); + nodes.addAll(this.serverNodeWrappers); + nodes.addAll(this.storeNodeWrappers); + nodes.addAll(this.pdNodeWrappers); + stopNodes(nodes, !this.startupFailed); + } + + static void stopNodes(List nodes, boolean deleteData) { + boolean interrupted = Thread.interrupted(); + IllegalStateException failure = null; + try { + for (AbstractNodeWrapper node : nodes) { + try { + node.stop(deleteData); + } catch (RuntimeException | Error e) { + if (failure == null) { + failure = new IllegalStateException("Failed to stop cluster nodes"); + } + failure.addSuppressed(e); + } finally { + interrupted |= Thread.interrupted(); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } } - for (PDNodeWrapper pdNodeWrapper : pdNodeWrappers) { - pdNodeWrapper.stop(); + if (failure != null) { + throw failure; } } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java index 8236bb1392..39d46e867e 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java @@ -28,6 +28,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; import java.util.Scanner; @@ -158,29 +159,61 @@ public boolean isStarted() { } public void stop() { + this.stop(true); + } + + public void stop(boolean deleteData) { if (this.instance == null) { return; } this.instance.destroy(); + if (!waitForExit(this.instance, 20)) { + this.instance.destroyForcibly(); + if (!waitForExit(this.instance, 10)) { + throw new IllegalStateException("Node " + this.getID() + + " survived forced stop; log: " + this.getLogPath()); + } + } + if (deleteData) { + deleteDir(); + } + } + + private static boolean waitForExit(Process process, long seconds) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(seconds); + boolean interrupted = false; try { - if (!this.instance.waitFor(20, TimeUnit.SECONDS)) { - this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS); + while (true) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0L) { + return !process.isAlive(); + } + try { + return process.waitFor(remaining, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.error("Waiting node to shutdown error.", e); } - deleteDir(); } public boolean isAlive() { - return this.instance.isAlive(); + return this.instance != null && this.instance.isAlive(); } protected ProcessBuilder runCmd(List startCmd, File stdoutFile) throws IOException { + if (stdoutFile.exists()) { + Path previous = Files.createTempFile(stdoutFile.toPath().toAbsolutePath().getParent(), + stdoutFile.getName() + ".previous-", ".log"); + Files.move(stdoutFile.toPath(), previous, StandardCopyOption.REPLACE_EXISTING); + } FileUtils.write(stdoutFile, String.join(" ", startCmd) + System.lineSeparator() + System.lineSeparator(), - StandardCharsets.UTF_8, true); + StandardCharsets.UTF_8, false); ProcessBuilder processBuilder = new ProcessBuilder(startCmd) .redirectOutput(ProcessBuilder.Redirect.appendTo(stdoutFile)) .redirectError(ProcessBuilder.Redirect.appendTo(stdoutFile)); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java index e16b96781e..4db3e1615b 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/ServerNodeWrapper.java @@ -40,6 +40,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -49,19 +51,48 @@ public class ServerNodeWrapper extends AbstractNodeWrapper { private static List hgJars = loadHgJarsOnce(); - public ServerNodeWrapper(int clusterIndex, int index) { + public ServerNodeWrapper(int clusterIndex, int index, int gremlinPort) { super(clusterIndex, index); this.fileNames = new ArrayList<>( List.of(LOG4J_FILE, GREMLIN_SERVER_FILE, GREMLIN_DRIVER_SETTING_FILE, REMOTE_SETTING_FILE, REMOTE_OBJECTS_SETTING_FILE)); this.workPath = SERVER_LIB_PATH; createNodeDir(Paths.get(SERVER_TEMPLATE_PATH), getNodePath() + CONF_DIR + File.separator); + configureGremlinPort(gremlinPort); this.fileNames = new ArrayList<>(List.of(EMPTY_SAMPLE_GROOVY_FILE, EXAMPLE_GROOVY_FILE)); - this.startLine = "INFO: [HttpServer] Started."; createNodeDir(Paths.get(SERVER_PACKAGE_PATH), getNodePath()); createLogDir(); } + @Override + public boolean isStarted() { + try { + List lines = Files.readAllLines(Paths.get(this.getLogPath())); + boolean gremlinStarted = lines.stream().anyMatch( + line -> line.contains("Channel started at port")); + boolean restStarted = lines.stream().anyMatch( + line -> line.contains("RestServer started")); + return gremlinStarted && restStarted; + } catch (IOException ignored) { + return false; + } + } + + private void configureGremlinPort(int gremlinPort) { + Path path = Paths.get(getNodePath(), CONF_DIR, GREMLIN_SERVER_FILE); + try { + String config = Files.readString(path); + String updated = config.replace("#port: 8182", + "port: " + gremlinPort); + if (updated.equals(config)) { + throw new IllegalStateException("Missing default Gremlin port"); + } + Files.writeString(path, updated); + } catch (IOException e) { + throw new AssertionError("Failed to configure Gremlin port", e); + } + } + private static void addJarsToClasspath(File directory, List classpath) { if (directory.exists() && directory.isDirectory()) { File[] files = directory.listFiles((dir, name) -> name.endsWith(".jar")); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java index 1cb0f67eae..868f4037eb 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/StoreNodeWrapper.java @@ -39,7 +39,7 @@ public StoreNodeWrapper() { super(); this.fileNames = new ArrayList<>(List.of(LOG4J_FILE)); this.workPath = STORE_LIB_PATH; - this.startLine = "o.a.h.s.n.StoreNodeApplication - Starting StoreNodeApplication"; + this.startLine = "StoreNodeApplication started."; createNodeDir(Paths.get(STORE_TEMPLATE_PATH), getNodePath() + CONF_DIR + File.separator); createLogDir(); } @@ -48,7 +48,7 @@ public StoreNodeWrapper(int clusterId, int index) { super(clusterId, index); this.fileNames = new ArrayList<>(List.of(LOG4J_FILE)); this.workPath = STORE_LIB_PATH; - this.startLine = "o.a.h.s.n.StoreNodeApplication - Starting StoreNodeApplication"; + this.startLine = "StoreNodeApplication started."; createNodeDir(Paths.get(STORE_TEMPLATE_PATH), getNodePath() + CONF_DIR + File.separator); createLogDir(); } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/env/NodeStartupTest.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/env/NodeStartupTest.java new file mode 100644 index 0000000000..e7d82205c8 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/env/NodeStartupTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.env; + +import java.util.Arrays; + +import org.apache.hugegraph.ct.node.AbstractNodeWrapper; +import org.apache.hugegraph.ct.node.BaseNodeWrapper; +import org.junit.Assert; +import org.junit.Test; + +public class NodeStartupTest { + + @Test + public void testLiveReadyNode() { + AbstractEnv.awaitStarted(new TestNode(true, true), 100); + } + + @Test + public void testExitedNodeCannotPassWithOldReadyMarker() { + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.awaitStarted(new TestNode(false, true), 100)); + Assert.assertTrue(failure.getMessage().contains("process exited")); + Assert.assertTrue(failure.getMessage().contains("test-node")); + Assert.assertTrue(failure.getMessage().contains("test-start.log")); + } + + @Test(timeout = 5000) + public void testMissingReadyMarkerTimesOut() { + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.awaitStarted(new TestNode(true, false), 10)); + Assert.assertTrue(failure.getMessage().contains("timed out")); + } + + @Test + public void testInterruptIsPreserved() { + Thread.currentThread().interrupt(); + try { + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.awaitStarted(new TestNode(true, false), 100)); + Assert.assertTrue(failure.getCause() instanceof InterruptedException); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + @Test + public void testCleanupContinuesAfterFailureAndPreservesDirectories() { + StopNode first = new StopNode(true); + StopNode second = new StopNode(false); + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> AbstractEnv.stopNodes(Arrays.asList(first, second), false)); + Assert.assertEquals(1, failure.getSuppressed().length); + Assert.assertTrue(first.stopped); + Assert.assertTrue(second.stopped); + Assert.assertFalse(first.deleteData); + Assert.assertFalse(second.deleteData); + } + + @Test + public void testCleanupPreservesInterruptAfterStoppingEveryNode() { + StopNode first = new StopNode(false); + StopNode second = new StopNode(false); + Thread.currentThread().interrupt(); + try { + AbstractEnv.stopNodes(Arrays.asList(first, second), false); + Assert.assertTrue(first.stopped); + Assert.assertTrue(second.stopped); + Assert.assertFalse(first.interruptedAtStop); + Assert.assertFalse(second.interruptedAtStop); + Assert.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + private static class StopNode extends AbstractNodeWrapper { + + private final boolean fail; + private boolean stopped; + private boolean deleteData; + private boolean interruptedAtStop; + + StopNode(boolean fail) { + this.fail = fail; + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } + + @Override + public String getID() { + return "stop-node"; + } + + @Override + public void stop(boolean deleteData) { + this.stopped = true; + this.deleteData = deleteData; + this.interruptedAtStop = Thread.currentThread().isInterrupted(); + if (this.fail) { + throw new IllegalStateException("test stop failure"); + } + } + } + + private static class TestNode implements BaseNodeWrapper { + + private final boolean alive; + private final boolean ready; + + TestNode(boolean alive, boolean ready) { + this.alive = alive; + this.ready = ready; + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } + + @Override + public void stop() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isAlive() { + return this.alive; + } + + @Override + public boolean isStarted() { + return this.ready; + } + + @Override + public String getID() { + return "test-node"; + } + + @Override + public String getNodePath() { + return "test-node/"; + } + + @Override + public String getLogPath() { + return "test-node/test-start.log"; + } + + @Override + public void updateWorkPath(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public void updateConfigPath(String path) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/NodeProcessTest.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/NodeProcessTest.java new file mode 100644 index 0000000000..14f9674192 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/test/java/org/apache/hugegraph/ct/node/NodeProcessTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.ct.node; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.Assert; +import org.junit.Test; + +public class NodeProcessTest { + + @Test + public void testNewCommandArchivesOldReadyMarker() throws Exception { + Path directory = Files.createTempDirectory("hugegraph-node-log-"); + try { + ProcessNode node = new ProcessNode(null, directory); + node.startLine = "OLD_READY"; + node.configPath = directory.toString(); + Path log = directory.resolve("start.log"); + Files.writeString(log, "OLD_READY\n"); + node.runCmd(Collections.singletonList("test-command"), log.toFile()); + Assert.assertEquals("test-command", Files.readString(log).trim()); + try (Stream files = Files.list(directory)) { + Path archive = files.filter(path -> path.getFileName().toString() + .startsWith("start.log.previous-")) + .findFirst().orElseThrow(AssertionError::new); + Assert.assertEquals("OLD_READY", Files.readString(archive).trim()); + } + } finally { + try (Stream files = Files.list(directory)) { + for (Path file : (Iterable) files::iterator) { + Files.delete(file); + } + } + Files.delete(directory); + } + } + + @Test(timeout = 40000) + public void testForcedStopPreservesFailureDirectory() throws Exception { + checkStop(true, false, false); + } + + @Test(timeout = 20000) + public void testGracefulStopDeletesNormalDirectory() throws Exception { + checkStop(false, true, false); + } + + @Test(timeout = 40000) + public void testInterruptDuringStopStillReapsChild() throws Exception { + checkStop(true, false, true); + } + + @Test + public void testSurvivingForcedStopFailsAndRetainsDirectory() throws Exception { + Path directory = Files.createTempDirectory("hugegraph-node-survivor-"); + try { + ProcessNode node = new ProcessNode(new StubbornProcess(), directory); + IllegalStateException failure = Assert.assertThrows(IllegalStateException.class, + () -> node.stop(true)); + Assert.assertTrue(failure.getMessage().contains("survived forced stop")); + Assert.assertTrue(Files.exists(directory)); + } finally { + Files.delete(directory); + } + } + + private static void checkStop(boolean slowShutdown, boolean deleteData, + boolean interrupt) throws Exception { + Path directory = Files.createTempDirectory("hugegraph-node-stop-"); + Process process = null; + Thread interrupter = null; + try { + process = new ProcessBuilder( + Paths.get(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", System.getProperty("java.class.path"), NodeProcessTest.class.getName(), + directory.toString(), Boolean.toString(slowShutdown)) + .redirectErrorStream(true).redirectOutput(directory.resolve("child.log").toFile()) + .start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!Files.exists(directory.resolve("ready"))) { + Assert.assertTrue("Child exited before ready", process.isAlive()); + Assert.assertTrue("Child readiness timed out", System.nanoTime() < deadline); + Thread.sleep(10); + } + ProcessNode node = new ProcessNode(process, directory); + if (interrupt) { + Thread caller = Thread.currentThread(); + interrupter = new Thread(() -> { + try { + Thread.sleep(100); + caller.interrupt(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + interrupter.start(); + } + node.stop(deleteData); + Assert.assertEquals(interrupt, Thread.currentThread().isInterrupted()); + Thread.interrupted(); + Assert.assertFalse("Node process survived stop", process.isAlive()); + Assert.assertEquals(!deleteData, Files.exists(directory)); + if (!deleteData) { + Assert.assertTrue(Files.exists(directory.resolve("child.log"))); + } + } finally { + Thread.interrupted(); + if (interrupter != null) { + interrupter.interrupt(); + interrupter.join(1000); + } + if (process != null && process.isAlive()) { + process.destroyForcibly(); + process.waitFor(5, TimeUnit.SECONDS); + } + if (Files.exists(directory)) { + Files.deleteIfExists(directory.resolve("ready")); + Files.deleteIfExists(directory.resolve("child.log")); + Files.delete(directory); + } + } + } + + public static void main(String[] args) throws Exception { + if (Boolean.parseBoolean(args[1])) { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + Thread.sleep(60000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + })); + } + Files.createFile(Paths.get(args[0], "ready")); + Thread.sleep(60000); + } + + private static class StubbornProcess extends Process { + + @Override + public OutputStream getOutputStream() { + return OutputStream.nullOutputStream(); + } + + @Override + public InputStream getInputStream() { + return InputStream.nullInputStream(); + } + + @Override + public InputStream getErrorStream() { + return InputStream.nullInputStream(); + } + + @Override + public int waitFor() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean waitFor(long timeout, TimeUnit unit) { + return false; + } + + @Override + public int exitValue() { + throw new IllegalThreadStateException("still alive"); + } + + @Override + public boolean isAlive() { + return true; + } + + @Override + public void destroy() { + } + + @Override + public Process destroyForcibly() { + return this; + } + } + + private static class ProcessNode extends AbstractNodeWrapper { + + private final Path directory; + + ProcessNode(Process process, Path directory) { + this.instance = process; + this.directory = directory; + } + + @Override + public void start() { + throw new UnsupportedOperationException(); + } + + @Override + public String getID() { + return "process-test"; + } + + @Override + public String getNodePath() { + return this.directory == null ? super.getNodePath() : this.directory.toString(); + } + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/LoggingBindingTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/LoggingBindingTest.java new file mode 100644 index 0000000000..1318a0a229 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/LoggingBindingTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph; + +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.LoggerFactory; +import org.slf4j.helpers.NOPLoggerFactory; + +public class LoggingBindingTest { + + @Test + public void testClusterDiagnosticsHaveLoggingBackend() { + Assert.assertFalse("Cluster diagnostics must not use a no-op logger", + LoggerFactory.getILoggerFactory() instanceof NOPLoggerFactory); + Assert.assertTrue(LoggerFactory.getLogger(LoggingBindingTest.class).isErrorEnabled()); + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java index 6e55cdd200..40d3be41fb 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.MultiClusterTest; +import org.apache.hugegraph.LoggingBindingTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -24,6 +25,7 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ + LoggingBindingTest.class, MultiClusterDeployTest.class, MultiClusterFileTest.class, }) diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java index f0f0c33461..d6df82bcfa 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/BaseSimpleTest.java @@ -74,7 +74,10 @@ public static void initEnv() { public static void clearEnv() throws InterruptedException { env.stopCluster(); Thread.sleep(2000); - client.close(); + if (client != null) { + client.close(); + client = null; + } } protected String execCmd(String[] cmds) throws IOException { diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java index 267e186f9e..345ee487ef 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterDeployTest.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import org.apache.hugegraph.pd.client.PDClient; import org.apache.hugegraph.pd.client.PDConfig; @@ -26,6 +27,8 @@ import org.junit.Assert; import org.junit.Test; +import jakarta.ws.rs.core.Response; + public class SimpleClusterDeployTest extends BaseSimpleTest { @Test @@ -113,4 +116,10 @@ public void testServerNode() { + "\"properties\":[]\n" + "}", 202); } + + @Test + public void testGremlinEndpoint() { + Response response = client.get("gremlin", Map.of("gremlin", "1 + 1")); + assertResponseStatus(200, response); + } } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java index 1cae2bcdba..9e5637ebe6 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterFileTest.java @@ -18,6 +18,10 @@ package org.apache.hugegraph.SimpleClusterTest; import java.io.File; +import java.io.Reader; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Properties; import org.junit.Assert; import org.junit.Test; @@ -45,4 +49,17 @@ public void checkServerNodeDir() { } } + @Test + public void checkServerMetadataPDPeers() throws Exception { + String expected = String.join(",", env.getPDGrpcAddrs()); + for (String nodeDir : env.getServerNodeDir()) { + Properties config = new Properties(); + try (Reader reader = Files.newBufferedReader( + Paths.get(nodeDir, "conf", "rest-server.properties"))) { + config.load(reader); + } + Assert.assertEquals(expected, config.getProperty("pd.peers")); + } + } + } diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java index 7f24d8b46c..671e51deee 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/SimpleClusterTest/SimpleClusterSuiteTest.java @@ -17,6 +17,7 @@ package org.apache.hugegraph.SimpleClusterTest; +import org.apache.hugegraph.LoggingBindingTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -24,6 +25,7 @@ @RunWith(Suite.class) @Suite.SuiteClasses({ + LoggingBindingTest.class, SimpleClusterDeployTest.class, SimpleClusterFileTest.class, }) diff --git a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java index 9dc69cbdb9..4e33f99151 100644 --- a/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java +++ b/hugegraph-commons/hugegraph-common/src/test/java/org/apache/hugegraph/unit/license/MachineInfoTest.java @@ -18,6 +18,7 @@ package org.apache.hugegraph.unit.license; import java.net.InetAddress; +import java.net.Inet6Address; import java.net.UnknownHostException; import java.util.List; import java.util.regex.Pattern; @@ -33,10 +34,6 @@ public class MachineInfoTest { "^(([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\.){3}" + "([01]?\\d\\d?|2[0-4]\\d|25[0-5])$" ); - private static final Pattern IPV6_PATTERN = Pattern.compile( - "^(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$" - ); - private static final Pattern MAC_PATTERN = Pattern.compile( "^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$" ); @@ -47,8 +44,7 @@ public class MachineInfoTest { public void testGetIpAddressList() { List ipAddressList = machineInfo.getIpAddress(); for (String ip : ipAddressList) { - Assert.assertTrue(IPV4_PATTERN.matcher(ip).matches() || - IPV6_PATTERN.matcher(ip).matches()); + Assert.assertTrue(isIpAddress(ip)); } Assert.assertEquals(ipAddressList, machineInfo.getIpAddress()); } @@ -67,8 +63,47 @@ public void testGetLocalAllInetAddress() { List addressList = machineInfo.getLocalAllInetAddress(); for (InetAddress address : addressList) { String ip = address.getHostAddress(); - Assert.assertTrue(IPV4_PATTERN.matcher(ip).matches() || - IPV6_PATTERN.matcher(ip).matches()); + Assert.assertTrue(isIpAddress(ip)); + } + } + + @Test + public void testScopedIpv6AddressValidation() throws UnknownHostException { + byte[] bytes = new byte[16]; + bytes[0] = 0x20; + bytes[1] = 0x01; + bytes[2] = 0x0d; + bytes[3] = (byte) 0xb8; + bytes[15] = 1; + Inet6Address scoped = Inet6Address.getByAddress(null, bytes, 7); + Assert.assertTrue(isIpAddress(scoped.getHostAddress())); + Inet6Address parsed = (Inet6Address) InetAddress.getByName(scoped.getHostAddress()); + Assert.assertArrayEquals(bytes, parsed.getAddress()); + Assert.assertEquals(7, parsed.getScopeId()); + Assert.assertTrue(isIpAddress("2001:db8::1")); + Assert.assertTrue(isIpAddress("192.0.2.1")); + } + + @Test + public void testRejectMalformedIpAddresses() { + for (String ip : new String[]{"localhost", "256.0.0.1", "2001:db8::gg", + "2001::db8::1", "localhost:8080"}) { + Assert.assertFalse(isIpAddress(ip)); + } + } + + private static boolean isIpAddress(String ip) { + if (IPV4_PATTERN.matcher(ip).matches()) { + return true; + } + // Only parse IPv6 literals here; never resolve a hostname through DNS. + if (!ip.contains(":")) { + return false; + } + try { + return InetAddress.getByName(ip) instanceof Inet6Address; + } catch (UnknownHostException e) { + return false; } } diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index 655860d407..077e77ffac 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -33,9 +33,9 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz -# 2nd stage: runtime env +# Shared runtime environment # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:11-jre-noble AS runtime LABEL maintainer="HugeGraph Docker Maintainers " @@ -56,21 +56,82 @@ RUN apt-get -q update \ dumb-init \ procps \ curl \ + util-linux \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-*/ /hugegraph-pd/ - # 2. Init docker script COPY hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh EXPOSE 8620 -VOLUME /hugegraph-pd HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8620/v1/health >/dev/null ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["./docker-entrypoint.sh"] + +# Topling runtime packages. The current native library requires glibc 2.38+. +FROM runtime AS topling-runtime + +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + libaio1t64 \ + libbz2-1.0 \ + libjemalloc2 \ + liblz4-1 \ + libsnappy1v5 \ + liburing2 \ + zlib1g \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Build the formal component-local Topling distribution on Linux amd64. +FROM topling-runtime AS build-topling + +COPY --from=build /pkg/ /pkg/ +WORKDIR /pkg +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + rsync \ + unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && PD_DIR=$(find /pkg/hugegraph-pd -maxdepth 1 -type d \ + -name 'apache-hugegraph-pd-*' ! -name '*-topling' \ + -print -quit) \ + && test -n "$PD_DIR" \ + && VERSION=${PD_DIR##*/apache-hugegraph-pd-} \ + && install-dist/scripts/build-topling-distribution.sh pd "$VERSION" + +# Topling image: contains both providers and selects ToplingDB by default. +# Build with: docker build --platform linux/amd64 --target topling ... +FROM topling-runtime AS topling + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="topling" +COPY --from=build-topling /pkg/hugegraph-pd/apache-hugegraph-pd-*-topling/ /hugegraph-pd/ +ENV HG_PD_ROCKSDB_PROVIDER="topling" \ + HG_PD_ENFORCE_PROVIDER_MARKER="true" +RUN mkdir -p /hugegraph-pd/pd_data /hugegraph-pd/topling-pd-data +VOLUME ["/hugegraph-pd/pd_data", "/hugegraph-pd/topling-pd-data"] + +# Keep the standard image as the final/default target for existing builds. +FROM runtime AS standard + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="standard" +COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-*/ /hugegraph-pd/ +ENV HG_PD_ROCKSDB_PROVIDER="rocksdb" +# Preserve the historical root volume for anonymous-volume upgrades. +VOLUME /hugegraph-pd diff --git a/hugegraph-pd/docs/development.md b/hugegraph-pd/docs/development.md index 514bd989a1..fc44151b79 100644 --- a/hugegraph-pd/docs/development.md +++ b/hugegraph-pd/docs/development.md @@ -93,8 +93,13 @@ mvn clean install -DskipTests ``` **Output**: -- JARs in each module's `target/` directory -- Distribution package: `hg-pd-dist/target/hugegraph-pd-.tar.gz` +- JARs in each module's `target/` directory. `hg-pd-service-.jar` + is the regular dependency JAR used by `hg-pd-test`; + `hg-pd-service--exec.jar` is the Spring Boot executable JAR. +- Distribution directory: `apache-hugegraph-pd-/`, containing only the + executable service JAR in `lib/`. The startup script continues to use `java -jar`; + the ToplingDB startup path uses the same executable JAR with `JarLauncher`. +- Distribution archive on Linux: `apache-hugegraph-pd-.tar.gz`. **Build Time**: 2-5 minutes (first build may take longer for dependency download) diff --git a/hugegraph-pd/hg-pd-core/pom.xml b/hugegraph-pd/hg-pd-core/pom.xml index 38837f1e7d..6cd5d0c3e9 100644 --- a/hugegraph-pd/hg-pd-core/pom.xml +++ b/hugegraph-pd/hg-pd-core/pom.xml @@ -49,7 +49,7 @@ org.rocksdb rocksdbjni - 6.29.5 + ${rocksdb.version} org.apache.hugegraph diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java index 3503d1ffc8..3479040d0c 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/StoreNodeService.java @@ -858,6 +858,10 @@ public synchronized void checkStoreStatus() { } catch (PDException e) { log.error("StoreNodeService updateClusterStatus exception {}", e); + // A failed read must not replace the previous state with the + // builder's initial Cluster_OK. REST refreshes a stale + // Cluster_Not_Ready from this method. + return; } this.clusterStats = builder.setTimestamp(System.currentTimeMillis()).build(); if (this.clusterStats.getState() != Metapb.ClusterState.Cluster_OK) { diff --git a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java index bd2e7a9e22..6f7ea817b5 100644 --- a/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java +++ b/hugegraph-pd/hg-pd-core/src/main/java/org/apache/hugegraph/pd/store/HgKVStoreImpl.java @@ -67,7 +67,7 @@ public void init(PDConfig config) { final Lock writeLock = this.readWriteLock.writeLock(); writeLock.lock(); try { - this.dbPath = config.getDataPath() + "/rocksdb/"; + this.dbPath = config.getDataPath() + "/rocksdb"; File file = new File(this.dbPath); if (!file.exists()) { try { diff --git a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh index 77af2af89d..630d5020ff 100755 --- a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh +++ b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh @@ -87,8 +87,30 @@ require_env "HG_PD_AUTH_SECRET_KEY" : "${HG_PD_GRPC_PORT:=8686}" : "${HG_PD_REST_PORT:=8620}" -: "${HG_PD_DATA_PATH:=/hugegraph-pd/pd_data}" : "${HG_PD_INITIAL_STORE_COUNT:=1}" +: "${HG_PD_ROCKSDB_PROVIDER:=rocksdb}" + +case "${HG_PD_ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) + log "ERROR: HG_PD_ROCKSDB_PROVIDER must be rocksdb or topling" + exit 2 + ;; +esac +if [[ -z "${HG_PD_DATA_PATH:-}" ]]; then + if [[ "${HG_PD_ROCKSDB_PROVIDER}" == "topling" ]]; then + HG_PD_DATA_PATH="$(pwd)/topling-pd-data" + else + HG_PD_DATA_PATH="$(pwd)/pd_data" + fi +fi +export TOPLINGDB_ROCKSDB_PROVIDER="${HG_PD_ROCKSDB_PROVIDER}" +./bin/verify-rocksdb-provider.sh \ + pd \ + "${HG_PD_ROCKSDB_PROVIDER}" \ + "${HG_PD_DATA_PATH}" \ + "${HG_PD_ENFORCE_PROVIDER_MARKER:-false}" + # Actuator endpoints reachable without a credential. Hardened by default; an # operator who needs /actuator/info or /actuator/loggers from this image opts # in deliberately instead of losing the endpoint. "*" is refused: /actuator/env @@ -119,6 +141,7 @@ SPRING_APPLICATION_JSON="$(cat <org.apache.hugegraph hg-pd-service ${revision} + exec diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml index c43a868352..1f6579ca13 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml @@ -41,6 +41,41 @@ + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh + + bin + common-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh + + bin + prepare-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh + + bin + preload-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh + + bin + verify-rocksdb-provider.sh + 0755 + + + @@ -49,7 +84,7 @@ runtime false - org.apache.hugegraph:${executable.jar.name}:jar:* + org.apache.hugegraph:${executable.jar.name}:jar:exec:* diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh index 1329df2271..f17c791888 100755 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -66,6 +66,13 @@ PID_FILE="$BIN/pid" ensure_path_writable "$LOGS" ensure_path_writable "$PLUGINS" +# preload rocksdb/toplingdb +if [ ! -r "$BIN/preload-topling.sh" ]; then + echo "Required RocksDB runtime selector not found: $BIN/preload-topling.sh" >&2 + exit 1 +fi +source "$BIN/preload-topling.sh" || exit 1 + # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) @@ -83,7 +90,7 @@ fi # check jdk version JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}') -if [[ $? -ne 0 || $JAVA_VERSION < $EXPECT_JDK_VERSION ]]; then +if [[ $? -ne 0 || $JAVA_VERSION -lt $EXPECT_JDK_VERSION ]]; then echo "Please make sure that the JDK is installed and the version >= $EXPECT_JDK_VERSION" >> ${OUTPUT} exit 1 fi @@ -163,22 +170,34 @@ fi #if [ "${JMX_EXPORT_PORT}" != "" ] && [ ${JMX_EXPORT_PORT} -ne 0 ] ; then # JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml" #fi -if [ $(ps -ef|grep -v grep| grep java|grep -cE ${CONF}) -ne 0 ]; then +if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then echo "HugeGraphPDServer is already running..." exit 0 fi JVM_OPTIONS="-Dlog4j.configurationFile=${CONF}/log4j2.xml -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" +BOOT_JARS=("${LIB}"/hg-pd-service-*.jar) +if [ "${#BOOT_JARS[@]}" -ne 1 ] || [ ! -f "${BOOT_JARS[0]}" ]; then + echo "Expected exactly one HugeGraph PD executable JAR in ${LIB}" >> "${OUTPUT}" + exit 1 +fi +BOOT_JAR="${BOOT_JARS[0]}" +JAVA_MAIN=(-jar "$BOOT_JAR") +if [ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]; then + JAVA_MAIN=(-cp "${TOPLING_RUNTIME_CLASSPATH}:${BOOT_JAR}" \ + org.springframework.boot.loader.JarLauncher) +fi # Turn on security check if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphPDServer in daemon mode..." if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar & + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" & else - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar >> ${OUTPUT} 2>&1 & + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 & fi PID="$!" # Write pid to file @@ -190,10 +209,11 @@ else echo "$$" > "$PID_FILE" echo "[+pid] $$" if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" else - exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar >> ${OUTPUT} 2>&1 + exec ${JAVA} -Dname="HugeGraphPD" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 fi fi diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh index b476d7935b..b42239d2b7 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -394,3 +394,21 @@ function kill_process_and_wait() { kill_process "$process_name" "$pid" wait_for_shutdown "$process_name" "$pid" "$timeout_s" } + +# Find HugeGraph server directory in parent path using prefix glob. +# Usage: find_hugegraph_server_dir "/path/to/parent" +# Returns: first matching directory path or empty string +function find_hugegraph_server_dir() { + local parent_dir="$1" + if [ -z "$parent_dir" ]; then + parent_dir="$(cd "${TOP:-$(pwd)}"/.. && pwd)" + fi + local found="" + for d in "$parent_dir"/apache-hugegraph-server*; do + if [ -d "$d" ]; then + found="$d" + break + fi + done + echo "$found" +} diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml index ada9791922..e1c4af7c8a 100644 --- a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml @@ -42,6 +42,10 @@ grpc: port: 8686 # The service address of grpc needs to be changed to the actual local IPv4 address when deploying. host: 127.0.0.1 +# rocksdb: +# provider: rocksdb +# option-path: ./conf/rocksdb_pd.yaml +# open-http: true server: # REST service port number diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml new file mode 100644 index 0000000000..cc87950bbe --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library/rocksdb_resource + listening_ports: '127.0.0.1:2012' + auto_start_http: false +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With + # memtable_as_log_index enabled, every batch write fails: + # "WriteBatch has no mmap wal". Keep the option false for this client. + # convert_to_sst is still Topling-specific. Do not open data written by + # this profile with the standard RocksDB provider; restore a snapshot + # taken before migration instead. + compaction_readahead_size: 0 + memtable_as_log_index: false diff --git a/hugegraph-pd/hg-pd-service/pom.xml b/hugegraph-pd/hg-pd-service/pom.xml index eef38b4073..d2aedec484 100644 --- a/hugegraph-pd/hg-pd-service/pom.xml +++ b/hugegraph-pd/hg-pd-service/pom.xml @@ -178,6 +178,8 @@ + + exec org.apache.hugegraph.pd.boot.HugePDServer diff --git a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java index bee39f23ed..d196c21f7d 100644 --- a/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java +++ b/hugegraph-pd/hg-pd-service/src/main/java/org/apache/hugegraph/pd/rest/IndexAPI.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.ExecutionException; +import org.apache.hugegraph.pd.StoreNodeService; import org.apache.hugegraph.pd.common.PDException; import org.apache.hugegraph.pd.grpc.Metapb; import org.apache.hugegraph.pd.grpc.Pdpb; @@ -65,7 +66,7 @@ public BriefStatistics index() throws PDException, ExecutionException, Interrupt BriefStatistics statistics = new BriefStatistics(); statistics.leader = RaftEngine.getInstance().getLeaderGrpcAddress(); - statistics.state = pdService.getStoreNodeService().getClusterStats().getState().toString(); + statistics.state = this.clusterState(); // Use pdService (consistent with cluster()) rather than RaftEngine directly CallStreamObserverWrap membersResp = @@ -99,6 +100,19 @@ public BriefStatistics index() throws PDException, ExecutionException, Interrupt } + /** + * Recompute cluster state from the replicated store list. Registration and + * heartbeats update the cache only on the raft leader, so a follower can + * keep either the constructor value {@code Cluster_Not_Ready} or a stale + * {@code Cluster_OK} from an earlier term. {@code /v1/ready} stays the raft + * probe and does not call this method. + */ + String clusterState() { + StoreNodeService stores = this.pdService.getStoreNodeService(); + stores.checkStoreStatus(); + return stores.getClusterStats().getState().toString(); + } + /** * Get cluster statistics * Obtain various statistics about the cluster by calling related services, including node status, member list, storage information, graph information, etc., @@ -109,13 +123,13 @@ public BriefStatistics index() throws PDException, ExecutionException, Interrupt * @throws ExecutionException If an exception occurs during task execution, this exception is thrown * @throws PDException If an exception occurs while processing cluster statistics, such as service call failure or data processing errors, a PDException exception is thrown */ + @GetMapping(value = "/v1/cluster", produces = MediaType.APPLICATION_JSON_VALUE) @ResponseBody public RestApiResponse cluster() throws InterruptedException, ExecutionException { Statistics statistics = new Statistics(); try { - statistics.state = - String.valueOf(pdService.getStoreNodeService().getClusterStats().getState()); + statistics.state = this.clusterState(); String leaderGrpcAddress = RaftEngine.getInstance().getLeaderGrpcAddress(); CallStreamObserverWrap response = new CallStreamObserverWrap<>(); diff --git a/hugegraph-pd/hg-pd-test/README.md b/hugegraph-pd/hg-pd-test/README.md new file mode 100644 index 0000000000..81fc525d3c --- /dev/null +++ b/hugegraph-pd/hg-pd-test/README.md @@ -0,0 +1,15 @@ +# PD module tests + +Run common and core suites separately so native and Raft singletons do not carry +state between suite processes: + +```bash +mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-common-test +mvn test -pl hugegraph-pd/hg-pd-test -am -P pd-core-test +``` + +The core fixture uses `tmp/pd-core-data` relative to the test module by default. +Override it with `-Dpd.test.data_path=/absolute/task-owned/path` when isolating a +validation run. The fixture deletes that directory before initialization; never +point it at an existing service's data. Client and REST suites also require the +PD service setup described in the repository CI workflow. diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java index 9e7b03d98e..cb56beae2b 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/core/PDCoreTestBase.java @@ -44,7 +44,8 @@ public class PDCoreTestBase { - private static final String DATA_PATH = "/tmp/pd_data"; + private static final String DATA_PATH = System.getProperty("pd.test.data_path", + "tmp/pd-core-data"); private static PDConfig pdConfig; private static StoreNodeService storeNodeService; private static PartitionService partitionService; diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/IndexAPIClusterStateTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/IndexAPIClusterStateTest.java new file mode 100644 index 0000000000..bc6cf234d8 --- /dev/null +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/rest/IndexAPIClusterStateTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.pd.rest; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.hugegraph.pd.StoreNodeService; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.pd.service.PDService; +import org.junit.Assert; +import org.junit.Test; + +public class IndexAPIClusterStateTest { + + @Test + public void testRestRecomputesBeforeReadingNotReady() { + StoreNodeService stores = mock(StoreNodeService.class); + when(stores.getClusterStats()).thenReturn(stats(Metapb.ClusterState.Cluster_Not_Ready)); + IndexAPI api = api(stores); + + Assert.assertEquals("Cluster_Not_Ready", api.clusterState()); + verify(stores, times(1)).checkStoreStatus(); + } + + @Test + public void testRestRecomputesBeforeReadingOk() { + StoreNodeService stores = mock(StoreNodeService.class); + when(stores.getClusterStats()).thenReturn(stats(Metapb.ClusterState.Cluster_OK)); + IndexAPI api = api(stores); + + Assert.assertEquals("Cluster_OK", api.clusterState()); + verify(stores, times(1)).checkStoreStatus(); + } + + private static IndexAPI api(StoreNodeService stores) { + PDService pdService = mock(PDService.class); + when(pdService.getStoreNodeService()).thenReturn(stores); + IndexAPI api = new IndexAPI(); + api.pdService = pdService; + return api; + } + + private static Metapb.ClusterStats stats(Metapb.ClusterState state) { + return Metapb.ClusterStats.newBuilder().setState(state).build(); + } +} diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 67866974cf..4a543c649a 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -33,9 +33,9 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz -# 2nd stage: runtime env +# Shared runtime environment # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:11-jre-noble AS runtime LABEL maintainer="HugeGraph Docker Maintainers " @@ -49,7 +49,7 @@ WORKDIR /hugegraph-server/ # 1. Install runtime dependencies # Note: iproute2 provides `ss`, which the bin/util.sh port preflight needs. The -# jammy base image ships neither ss nor netstat, so without it the preflight is +# The base image ships neither ss nor netstat, so without it the preflight is # permanently inconclusive and a duplicate start is no longer caught. It # replaces lsof, which no shipped script calls any more. ARG RUNTIME_DEPS_EPOCH=1 @@ -59,24 +59,91 @@ RUN apt-get -q update \ procps \ curl \ iproute2 \ + util-linux \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ -RUN sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" ./conf/rest-server.properties - # 2. Init docker script -COPY hugegraph-server/hugegraph-dist/docker/scripts/remote-connect.groovy ./scripts -COPY hugegraph-server/hugegraph-dist/docker/scripts/detect-storage.groovy ./scripts +COPY hugegraph-server/hugegraph-dist/docker/scripts/remote-connect.groovy /opt/hugegraph-docker/ +COPY hugegraph-server/hugegraph-dist/docker/scripts/detect-storage.groovy /opt/hugegraph-docker/ COPY hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh EXPOSE 8080 -VOLUME /hugegraph-server HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8080/versions >/dev/null ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["./docker-entrypoint.sh"] + +# Topling runtime packages. The current native library requires glibc 2.38+. +FROM runtime AS topling-runtime + +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + libaio1t64 \ + libbz2-1.0 \ + libjemalloc2 \ + liblz4-1 \ + libsnappy1v5 \ + liburing2 \ + zlib1g \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Build the formal Server Topling distribution on Linux amd64. +FROM topling-runtime AS build-topling + +COPY --from=build /pkg/ /pkg/ +WORKDIR /pkg +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + rsync \ + unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && SERVER_DIR=$(find /pkg/hugegraph-server -maxdepth 1 -type d \ + -name 'apache-hugegraph-server-*' ! -name '*-topling' \ + -print -quit) \ + && test -n "$SERVER_DIR" \ + && VERSION=${SERVER_DIR##*/apache-hugegraph-server-} \ + && install-dist/scripts/build-topling-distribution.sh server "$VERSION" + +# ToplingDB image: contains both providers and selects ToplingDB by default. +# Build with: docker build --platform linux/amd64 --target topling ... +FROM topling-runtime AS topling + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="topling" +COPY --from=build-topling /pkg/hugegraph-server/apache-hugegraph-server-*-topling/ /hugegraph-server/ +RUN cp /opt/hugegraph-docker/*.groovy ./scripts/ \ + && sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" \ + ./conf/rest-server.properties +ENV HG_SERVER_ROCKSDB_PROVIDER="topling" +ENV HG_SERVER_ENFORCE_PROVIDER_MARKER="true" +RUN mkdir -p /hugegraph-server/rocksdb-data /hugegraph-server/topling-data +VOLUME ["/hugegraph-server/rocksdb-data", "/hugegraph-server/topling-data"] + +# Keep the standard image as the final/default target for existing builds. +FROM runtime AS standard + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="standard" +COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ +RUN cp /opt/hugegraph-docker/*.groovy ./scripts/ \ + && sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" \ + ./conf/rest-server.properties +ENV HG_SERVER_ROCKSDB_PROVIDER="rocksdb" +# Preserve the historical root volume so existing anonymous-volume upgrades +# continue to expose their data. New named-volume examples use rocksdb-data. +VOLUME /hugegraph-server diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index e90979563b..8517621110 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -61,6 +61,12 @@ RUN apt-get -q update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" \ + org.apache.hugegraph.rocksdb-runtime="hstore" + COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ # Configure the default HStore backend and REST listen address RUN cd /hugegraph-server/conf/graphs \ diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java index bb954446db..80f5b87436 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeFactoryAuthProxy.java @@ -185,7 +185,7 @@ private static void registerPrivateActions() { "this$0"); Reflection.registerFieldsToFilter(HugeGraphAuthProxy.Context.class, "ADMIN", "user"); Reflection.registerFieldsToFilter(HugeGraphAuthProxy.ContextTask.class, "runner", - "context"); + "cleanup", "context"); Reflection.registerFieldsToFilter(StandardHugeGraph.class, "LOG", "started", "closed", "mode", "variables", "name", "params", "configuration", "schemaEventHub", "graphEventHub", "indexEventHub", @@ -203,7 +203,8 @@ private static void registerPrivateActions() { "access$14", "access$15", "access$16", "access$17", "access$18", "serializer", "loadSchemaStore", "loadSystemStore", "loadGraphStore", "closeTx", - "analyzer", "serverInfoManager", "reloadRamtable", + "closeCurrentThreadTransaction", "analyzer", + "serverInfoManager", "reloadRamtable", "reloadRamtable", "access$19", "access$20", "access$21"); Reflection.registerFieldsToFilter( loadClass("org.apache.hugegraph.StandardHugeGraph$StandardHugeGraphParams"), @@ -298,8 +299,10 @@ private static void registerPrivateActions() { "autoCommit", "beforeRead", "afterWrite", "afterRead", "commitMutation2Backend", "checkOwnerThread", "doAction", "store", "reset"); - Reflection.registerFieldsToFilter(HugeFactory.class, "LOG", "NAME_REGEX", "graphs"); - Reflection.registerMethodsToFilter(HugeFactory.class, "lambda$0"); + Reflection.registerFieldsToFilter(HugeFactory.class, "LOG", "NAME_REGEX", "graphs", + "GRAPHS"); + Reflection.registerMethodsToFilter(HugeFactory.class, "lambda$0", + "closeCurrentThreadTransactions"); Reflection.registerFieldsToFilter(SchemaElement.class, "graph", "id", "name", "userdata", "status"); Reflection.registerFieldsToFilter(HugeVertex.class, "EMPTY_SET", "id", "label", "edges", diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java index 9ecda6b9f7..54e571849f 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/auth/HugeGraphAuthProxy.java @@ -40,6 +40,7 @@ import javax.security.sasl.AuthenticationException; import org.apache.commons.configuration2.Configuration; +import org.apache.hugegraph.HugeFactory; import org.apache.hugegraph.HugeGraph; import org.apache.hugegraph.auth.HugeAuthenticator.RolePerm; import org.apache.hugegraph.auth.HugeAuthenticator.User; @@ -1289,11 +1290,17 @@ public User user() { static class ContextTask implements Runnable { private final Runnable runner; + private final Runnable cleanup; private final Context context; public ContextTask(Runnable runner) { + this(runner, HugeFactory::closeCurrentThreadTransactions); + } + + ContextTask(Runnable runner, Runnable cleanup) { this.context = getContext(); this.runner = runner; + this.cleanup = cleanup; } @Override @@ -1302,7 +1309,13 @@ public void run() { try { this.runner.run(); } finally { - resetContext(); + try { + this.cleanup.run(); + } catch (Throwable e) { + LOG.error("Failed to close Gremlin worker transactions", e); + } finally { + resetContext(); + } } } } diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/server/ApplicationConfig.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/server/ApplicationConfig.java index dd4a0f46e9..adf22442d0 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/server/ApplicationConfig.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/server/ApplicationConfig.java @@ -18,11 +18,13 @@ package org.apache.hugegraph.server; import org.apache.hugegraph.HugeException; +import org.apache.hugegraph.HugeFactory; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.core.GraphManager; import org.apache.hugegraph.define.WorkLoad; import org.apache.hugegraph.event.EventHub; import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.Log; import org.apache.hugegraph.version.CoreVersion; import org.apache.tinkerpop.gremlin.server.util.MetricManager; import org.glassfish.hk2.api.Factory; @@ -35,6 +37,7 @@ import org.glassfish.jersey.server.monitoring.ApplicationEventListener; import org.glassfish.jersey.server.monitoring.RequestEvent; import org.glassfish.jersey.server.monitoring.RequestEventListener; +import org.slf4j.Logger; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.jersey3.InstrumentedResourceMethodApplicationListener; @@ -64,6 +67,8 @@ ) @ApplicationPath("/") public class ApplicationConfig extends ResourceConfig { + private static final Logger LOG = Log.logger(ApplicationConfig.class); + @Context private ServletConfig servletConfig; @@ -195,7 +200,15 @@ public void onEvent(ApplicationEvent event) { @Override public RequestEventListener onRequest(RequestEvent event) { - return null; + return request -> { + if (request.getType() == RequestEvent.Type.FINISHED) { + try { + HugeFactory.closeCurrentThreadTransactions(); + } catch (RuntimeException e) { + LOG.error("Failed to release REST request transactions", e); + } + } + }; } }); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java index f258cb5c7b..e0b4906f39 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/HugeFactory.java @@ -19,7 +19,10 @@ import java.io.File; import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; @@ -104,11 +107,39 @@ public static HugeGraph open(URL url) { return open(getRemoteConfig(url)); } - public static void remove(HugeGraph graph) { + public static synchronized void remove(HugeGraph graph) { String spaceGraphName = graph.spaceGraphName(); GRAPHS.remove(spaceGraphName); } + public static void closeCurrentThreadTransactions() { + List graphs; + synchronized (HugeFactory.class) { + graphs = new ArrayList<>(GRAPHS.values()); + } + closeCurrentThreadTransactions(graphs); + } + + static void closeCurrentThreadTransactions( + Collection graphs) { + Throwable failure = null; + for (HugeGraph graph : graphs) { + try { + ((StandardHugeGraph) graph).closeCurrentThreadTransaction(); + } catch (Throwable e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw new HugeException("Failed to close current thread " + + "transactions", failure); + } + } + public static void checkGraphName(String name, String configFile) { E.checkArgument(SYS_GRAPH.equals(name) || name.matches(NAME_REGEX), "Invalid graph name '%s' in %s, " + diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java index 7f79c93bb7..17d1020ab5 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java @@ -393,6 +393,10 @@ private void closeTx() { } } + void closeCurrentThreadTransaction() { + this.closeTx(); + } + @Override public GraphMode mode() { return this.mode; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendSessionPool.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendSessionPool.java index 521f44556f..44137cd766 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendSessionPool.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendSessionPool.java @@ -56,14 +56,17 @@ public HugeConfig config() { public final BackendSession getOrNewSession() { BackendSession session = this.threadLocalSession.get(); if (session == null) { - session = this.newSession(); - assert session != null; - this.threadLocalSession.set(session); - assert !this.sessions.containsKey(Thread.currentThread().getId()); - this.sessions.put(Thread.currentThread().getId(), session); - int sessionCount = this.sessionCount.incrementAndGet(); - LOG.debug("Now(after connect({})) session count is: {}", - this, sessionCount); + // Serialize new borrowers with the last-session native close. + synchronized (this) { + session = this.newSession(); + assert session != null; + this.threadLocalSession.set(session); + assert !this.sessions.containsKey(Thread.currentThread().getId()); + this.sessions.put(Thread.currentThread().getId(), session); + int sessionCount = this.sessionCount.incrementAndGet(); + LOG.debug("Now(after connect({})) session count is: {}", + this, sessionCount); + } } else { this.detectSession(session); } @@ -131,7 +134,7 @@ public void forceResetSessions() { } } - public boolean close() { + public synchronized boolean close() { Pair result = Pair.of(-1, -1); try { result = this.closeSession(); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java index 6faace9671..1f9f314d28 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphIndexTransaction.java @@ -168,7 +168,7 @@ public void updateVertexIndex(HugeVertex vertex, boolean removed) { return; } // Update index(only property, no edge) of a vertex - for (Id id : vertex.schemaLabel().indexLabels()) { + for (Id id : this.indexLabelIds(vertex.schemaLabel())) { this.updateIndex(id, vertex, removed); } } @@ -176,7 +176,7 @@ public void updateVertexIndex(HugeVertex vertex, boolean removed) { @Watched(prefix = "index") public void updateEdgeIndex(HugeEdge edge, boolean removed) { // Update index of an edge - for (Id id : edge.schemaLabel().indexLabels()) { + for (Id id : this.indexLabelIds(edge.schemaLabel())) { this.updateIndex(id, edge, removed); } @@ -857,6 +857,9 @@ private MatchedIndex collectMatchedIndex(SchemaLabel schemaLabel, private ConditionQuery constructSearchQuery(ConditionQuery query, MatchedIndex index) { ConditionQuery newQuery = query; + // Keep this incoming branch's non-search conditions. Expanding an EQ + // branch back to its root IN condition can return the same element + // once per branch when joint indexes fall back to filtering. Set indexFields = new HashSet<>(); // Convert has(key, text) to has(key, textContainsAny(word1, word2)) for (IndexLabel il : index.indexLabels()) { @@ -1557,13 +1560,27 @@ private static void validateIndexLabel(IndexLabel indexLabel) { indexLabel, indexLabel.status()); } - private static boolean hasNullableProp(HugeElement element, Id key) { - return element.schemaLabel().nullableKeys().contains(key); + private boolean hasNullableProp(HugeElement element, Id key) { + return this.currentSchemaLabel(element.schemaLabel()).nullableKeys().contains(key); } - private static Set relatedIndexLabels(HugeElement element) { + Set indexLabelIds(SchemaLabel label) { + return this.currentSchemaLabel(label).indexLabels(); + } + + private SchemaLabel currentSchemaLabel(SchemaLabel label) { + // Elements can outlive schema cache eviction. Resolve the current index + // membership rather than using the schema object retained by an element. + ISchemaTransaction schema = this.params().schemaTransaction(); + SchemaLabel current = label.type() == HugeType.VERTEX_LABEL ? + schema.getVertexLabel(label.id()) : schema.getEdgeLabel(label.id()); + E.checkArgument(current != null, "Not exist schema label with id '%s'", label.id()); + return current; + } + + private Set relatedIndexLabels(HugeElement element) { Set indexLabels = InsertionOrderUtil.newSet(); - Set indexLabelIds = element.schemaLabel().indexLabels(); + Set indexLabelIds = this.indexLabelIds(element.schemaLabel()); for (Id id : indexLabelIds) { IndexLabel indexLabel = element.graph().indexLabel(id); @@ -1870,7 +1887,7 @@ private long processSecondaryOrSearchIndexLeft(ConditionQuery query, // Delete unused index long count = 0; Set incorrectPkIds; - for (IndexLabel il : relatedIndexLabels(deletion)) { + for (IndexLabel il : this.tx.relatedIndexLabels(deletion)) { incorrectPkIds = incorrectPKs.keySet().stream() .map(PropertyKey::id) .collect(Collectors.toSet()); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index b22fb1dad3..bd22d1abb9 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -369,7 +369,12 @@ protected void prepareDeletions(Map removedVertices, Map removedEdges) { // Remove related edges of each vertex for (HugeVertex v : removedVertices.values()) { - if (!v.schemaLabel().existsLinkLabel()) { + // System edge labels may be absent from schema enumeration after + // request cleanup, so always scan auth/system vertex relations. + // OLAP shares its ID with a base vertex and must keep its original + // path rather than scan that base vertex's ordinary edges. + VertexLabel label = v.schemaLabel(); + if ((!label.system() || label.olap()) && !label.existsLinkLabel()) { continue; } // Query all edges of the vertex and remove them @@ -621,7 +626,7 @@ public HugeVertex addVertex(HugeVertex vertex) { this.locksTable.lockReads(LockUtil.VERTEX_LABEL_DELETE, vertex.schemaLabel().id()); this.locksTable.lockReads(LockUtil.INDEX_LABEL_DELETE, - vertex.schemaLabel().indexLabels()); + this.indexTx.indexLabelIds(vertex.schemaLabel())); // Ensure vertex label still exists from vertex-construct to lock this.graph().vertexLabel(vertex.schemaLabel().id()); /* @@ -694,7 +699,7 @@ public void removeVertex(HugeVertex vertex) { // Override vertices in local `addedVertices` this.addedVertices.remove(vertex.id()); // Force load vertex to ensure all properties are loaded (refer to #2181) - if (!vertex.schemaLabel().indexLabels().isEmpty()) { + if (!this.indexTx.indexLabelIds(vertex.schemaLabel()).isEmpty()) { vertex.forceLoad(); } // Collect the removed vertex @@ -743,7 +748,7 @@ public Iterator queryTaskInfos(Query query) { } public Iterator queryTaskInfos(Object... vertexIds) { - if (this.graph().backendStoreFeatures().supportsTaskAndServerVertex()) { + if (this.storeFeatures().supportsTaskAndServerVertex()) { return this.queryVerticesByIds(vertexIds, false, false, HugeType.TASK); } @@ -888,7 +893,7 @@ public HugeEdge addEdge(HugeEdge edge) { this.locksTable.lockReads(LockUtil.EDGE_LABEL_DELETE, edge.schemaLabel().id()); this.locksTable.lockReads(LockUtil.INDEX_LABEL_DELETE, - edge.schemaLabel().indexLabels()); + this.indexTx.indexLabelIds(edge.schemaLabel())); // Ensure edge label still exists from edge-construct to lock this.graph().edgeLabel(edge.schemaLabel().id()); /* @@ -1827,7 +1832,7 @@ private void lockForUpdateProperty(SchemaLabel schemaLabel, Id pkey = prop.propertyKey().id(); Set indexIds = new HashSet<>(); - for (Id il : schemaLabel.indexLabels()) { + for (Id il : this.indexTx.indexLabelIds(schemaLabel)) { if (graph().indexLabel(il).indexFields().contains(pkey)) { indexIds.add(il); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java index 204320d400..1f695c9107 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeSecurityManager.java @@ -63,7 +63,11 @@ public class HugeSecurityManager extends SecurityManager { "line.separator", "file.separator", // Sofa - "java.specification.version" + "java.specification.version", + // gRPC Netty reads these when a transport starts. Denying them + // under the Gremlin sandbox panics the shared channel. + "io.grpc.netty.shaded.io.grpc.netty.useCustomAllocator", + "io.grpc.netty.shaded.io.netty.allocator.maxOrder" ); private static final Map> ASYNC_TASKS = ImmutableMap.of( diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java index 3367572580..de5bb5ba55 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/task/StandardTaskScheduler.java @@ -596,7 +596,7 @@ private Iterator> queryTask(Map conditions, boolean withResult) { return this.call(() -> { ConditionQuery query; - if (this.graph.backendStoreFeatures().supportsTaskAndServerVertex()) { + if (this.tx().storeFeatures().supportsTaskAndServerVertex()) { query = new ConditionQuery(HugeType.TASK); } else { query = new ConditionQuery(HugeType.VERTEX); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java index 44f9cf8692..e118f033de 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/algorithm/OltpTraverser.java @@ -43,6 +43,7 @@ import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.Property; +import org.apache.tinkerpop.gremlin.structure.Transaction; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.util.CloseableIterator; @@ -73,7 +74,10 @@ protected OltpTraverser(HugeGraph graph) { @Override public void close() { - // pass + Transaction tx = this.graph().tx(); + if (tx.isOpen()) { + tx.close(); + } } public static void destroy() { diff --git a/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java b/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java index 5b1e32e5d6..d60e9361c3 100644 --- a/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java +++ b/hugegraph-server/hugegraph-core/src/test/java/org/apache/hugegraph/security/HugeSecurityManagerTest.java @@ -36,5 +36,9 @@ public void testWhiteSystemPropertiesExcludeRemovedBackends() throws Exception { Assert.assertFalse(properties.contains("socksProxyHost")); Assert.assertFalse(properties.contains("file.encoding")); Assert.assertTrue(properties.contains("java.specification.version")); + Assert.assertTrue(properties.contains( + "io.grpc.netty.shaded.io.grpc.netty.useCustomAllocator")); + Assert.assertTrue(properties.contains( + "io.grpc.netty.shaded.io.netty.allocator.maxOrder")); } } diff --git a/hugegraph-server/hugegraph-dist/docker/README.md b/hugegraph-server/hugegraph-dist/docker/README.md index aa76092b82..7de1360f5b 100644 --- a/hugegraph-server/hugegraph-dist/docker/README.md +++ b/hugegraph-server/hugegraph-dist/docker/README.md @@ -27,6 +27,30 @@ Use Docker to quickly start a standalone HugeGraph Server with RocksDB. - 8080:8080 ``` +### ToplingDB image + +The same Server Dockerfile has a `topling` target. It copies the checked-in +ToplingDB JAR into the Server distribution and prepares its native runtime at +image build time, so starting a container needs no GitHub token or download. + +```bash +docker build --platform linux/amd64 --target topling \ + -f hugegraph-server/Dockerfile \ + -t hugegraph/hugegraph:1.8.0-topling . +docker run -itd --name=graph -p 8080:8080 \ + hugegraph/hugegraph:1.8.0-topling +``` + +The `-topling` image selects ToplingDB by default. Set +`HG_SERVER_ROCKSDB_PROVIDER=rocksdb` to use the standard RocksDB provider from +the same image. ToplingDB currently supports only Linux x86-64. Do not switch +an existing data volume between providers without a supported migration. +Before changing providers, stop writes and take a verified snapshot of the +current volume. Start the target provider with a new empty volume, or restore a +snapshot that was created by that provider. A provider setting change is not a +data migration; never mount a ToplingDB data directory into standard RocksDB +for rollback. + ## 2. Create Sample Graph on Server Startup To preload sample data on startup, set `PRELOAD=true`. @@ -159,3 +183,11 @@ docker run -itd --name=graph -p 8080:8080 -e HG_SERVER_STARTUP_TIMEOUT_S=450 hug Raising it does not move the health check above. The images set `--interval=15s --start-period=90s --retries=3`, so a container given a longer startup budget is reported `unhealthy` around 135 seconds while the entrypoint is still legitimately waiting; raise it with `--health-start-period` on `docker run`. The Compose files replace those values with their own (`start_period: 60s`, `interval: 10s`, `retries: 30`, so roughly 360 seconds), and anything gated on `depends_on: condition: service_healthy`, Hubble included, waits on that budget rather than on this variable. Move the two together. + +### Offline store dump output + +After stopping a standalone RocksDB Server, `bin/dump-store.sh` uses the +packaged `conf/log4j2.xml` configuration to print the selected table and entries. +No external `JAVA_TOOL_OPTIONS` setting is required. If an operator changes the +logging configuration, keep INFO output enabled for +`org.apache.hugegraph.cmd.StoreDumper` to see the dump contents. diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index 42137dc3b5..68aef876e6 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -22,7 +22,11 @@ TEST_HOME=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-entrypoint-test.XXXXXX") trap 'rm -rf "${TEST_HOME}"' EXIT mkdir -p "${TEST_HOME}/bin" "${TEST_HOME}/conf/graphs" "${TEST_HOME}/docker" +mkdir -p "${TEST_HOME}/rocksdb-data" cp "${SCRIPT_DIR}/docker-entrypoint.sh" "${TEST_HOME}/docker-entrypoint.sh" +cp "${SCRIPT_DIR}/../src/assembly/static/bin/rocksdb-server-config.sh" "${TEST_HOME}/bin/" +cp "${SCRIPT_DIR}/../src/assembly/static/bin/verify-rocksdb-provider.sh" \ + "${TEST_HOME}/bin/verify-rocksdb-provider.sh" touch "${TEST_HOME}/docker/init_complete" cat > "${TEST_HOME}/conf/rest-server.properties" <<'EOF' @@ -39,6 +43,19 @@ cat > "${TEST_HOME}/bin/start-hugegraph.sh" <<'EOF' # than substring-matching a flattened "$*", where -t 1200 contains -t 120. printf '%s\n' "$@" > ./docker/start-hugegraph-argv printf 'called\n' >> ./docker/start-hugegraph-calls +if [[ "${START_TEST_PRE_PID_DELAY:-false}" == "true" ]]; then + touch ./docker/start-before-pid + sleep 300 +fi +if [[ "${START_TEST_CHILD:-false}" == "true" ]]; then + sleep 300 & + child_pid=$! + printf '%s\n' "$child_pid" > ./bin/pid + if [[ "${START_TEST_DELAY:-false}" == "true" ]]; then + trap 'kill -TERM "$child_pid" 2>/dev/null || true; wait "$child_pid" 2>/dev/null || true; exit 0' TERM + wait "$child_pid" + fi +fi exit 0 EOF cat > "${TEST_HOME}/bin/init-store.sh" <<'EOF' @@ -65,6 +82,7 @@ chmod +x "${TEST_HOME}/bin/"*.sh ( cd "${TEST_HOME}" HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ HG_SERVER_PD_PEERS=pd:8686 \ HG_SERVER_CLUSTER=hg \ HG_SERVER_USE_PD=true \ @@ -76,6 +94,8 @@ chmod +x "${TEST_HOME}/bin/"*.sh [[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 1 ]] grep -qx 'backend=hstore' "${TEST_HOME}/conf/graphs/hugegraph.properties" +grep -qx 'rocksdb.provider=rocksdb' \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" grep -qx 'pd.peers=pd:8686' "${TEST_HOME}/conf/graphs/hugegraph.properties" grep -qx 'usePD=true' "${TEST_HOME}/conf/rest-server.properties" grep -qx 'pd.peers=pd:8686' "${TEST_HOME}/conf/rest-server.properties" @@ -89,6 +109,78 @@ grep -qx 'auth.token_secret=12345678901234567890123456789012' \ grep -qx 'auth.token_secret=12345678901234567890123456789012' \ "${TEST_HOME}/conf/graphs/hugegraph.properties" +cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties.before-hstore-topling" +if ( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=topling \ + bash ./docker-entrypoint.sh +); then + echo "HStore Server unexpectedly selected a local Topling provider" >&2 + exit 1 +fi +cmp "${TEST_HOME}/conf/graphs/hugegraph.properties.before-hstore-topling" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" + +mkdir -p "${TEST_HOME}/lib/topling" "${TEST_HOME}/library" \ + "${TEST_HOME}/topling-data" +touch "${TEST_HOME}/lib/topling/rocksdbjni-topling.jar" +touch "${TEST_HOME}/library/librocksdbjni-linux64.so" +if ( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +); then + echo "HStore Server unexpectedly accepted a local Topling payload" >&2 + exit 1 +fi +rm -f "${TEST_HOME}/lib/topling/rocksdbjni-topling.jar" +rm -f "${TEST_HOME}/library/librocksdbjni-linux64.so" + +cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties.before-invalid-provider" +if ( + cd "${TEST_HOME}" + HG_SERVER_ROCKSDB_PROVIDER=invalid bash ./docker-entrypoint.sh +); then + echo "invalid RocksDB provider unexpectedly succeeded" >&2 + exit 1 +fi +cmp "${TEST_HOME}/conf/graphs/hugegraph.properties.before-invalid-provider" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" + +( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=rocksdb \ + HG_SERVER_ROCKSDB_PROVIDER=topling \ + HG_SERVER_ENFORCE_PROVIDER_MARKER=true \ + bash ./docker-entrypoint.sh +) +grep -qx "rocksdb.data_path=${TEST_HOME}/topling-data/data" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" +grep -qx "rocksdb.wal_path=${TEST_HOME}/topling-data/wal" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" +grep -Fqx 'component=server' \ + "${TEST_HOME}/topling-data/.hugegraph-rocksdb-provider" +grep -Fqx 'provider=topling' \ + "${TEST_HOME}/topling-data/.hugegraph-rocksdb-provider" +cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties.before-provider-mismatch" +if ( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=rocksdb \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + HG_SERVER_DATA_PATH="${TEST_HOME}/topling-data" \ + bash ./docker-entrypoint.sh +); then + echo "provider-mismatched Server data path unexpectedly succeeded" >&2 + exit 1 +fi +cmp "${TEST_HOME}/conf/graphs/hugegraph.properties.before-provider-mismatch" \ + "${TEST_HOME}/conf/graphs/hugegraph.properties" + cp "${TEST_HOME}/conf/rest-server.properties" \ "${TEST_HOME}/conf/rest-server.properties.before-short-secret" cp "${TEST_HOME}/conf/graphs/hugegraph.properties" \ @@ -117,7 +209,7 @@ if ( echo "required authentication token secret unexpectedly succeeded" >&2 exit 1 fi -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 1 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 2 ]] [[ ! -e "${TEST_HOME}/docker/enable-auth-calls" ]] ( @@ -125,7 +217,7 @@ fi HG_SERVER_REQUIRE_AUTH_TOKEN_SECRET=true \ bash ./docker-entrypoint.sh ) -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 2 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 3 ]] [[ ! -e "${TEST_HOME}/docker/enable-auth-calls" ]] ( @@ -135,7 +227,7 @@ fi HG_SERVER_AUTH_TOKEN_SECRET=12345678901234567890123456789012 \ bash ./docker-entrypoint.sh ) -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 3 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 4 ]] [[ "$(wc -l < "${TEST_HOME}/docker/enable-auth-calls")" -eq 1 ]] sed -i '/^auth\.token_secret=/d' "${TEST_HOME}/conf/rest-server.properties" @@ -181,7 +273,7 @@ sed -i "s|^auth\\.token_secret=.*|auth.token_secret ${rest_secret}|" \ grep -qx "auth.token_secret=${rest_secret}" \ "${TEST_HOME}/conf/rest-server.properties" -[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 7 ]] +[[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -eq 8 ]] [[ "$(wc -l < "${TEST_HOME}/docker/enable-auth-calls")" -eq 5 ]] ( @@ -230,12 +322,118 @@ grep -Fqx 'auth.admin_pa=Strong\\Pass\ 9!' \ "${TEST_HOME}/conf/rest-server.properties" rm -f "${TEST_HOME}/docker/init_complete" +rm -f "${TEST_HOME}/rocksdb-data/.hugegraph-state/init_complete" ( cd "${TEST_HOME}" PASSWORD=-n bash ./docker-entrypoint.sh ) grep -Fqx -- '-n' "${TEST_HOME}/docker/init-store-password" +rm -f "${TEST_HOME}/bin/pid" +( + cd "${TEST_HOME}" + exec setsid env \ + START_TEST_CHILD=true \ + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +) & +entrypoint_pid=$! +for _ in $(seq 1 10); do + [[ -s "${TEST_HOME}/bin/pid" ]] && break + sleep 1 +done +[[ -s "${TEST_HOME}/bin/pid" ]] +child_pid=$(<"${TEST_HOME}/bin/pid") +kill -TERM -- "-${entrypoint_pid}" +for _ in $(seq 1 15); do + ! kill -0 "${entrypoint_pid}" 2>/dev/null && break + sleep 1 +done +if kill -0 "${entrypoint_pid}" 2>/dev/null; then + kill -KILL -- "-${entrypoint_pid}" 2>/dev/null || true + wait "${entrypoint_pid}" 2>/dev/null || true + echo "Docker entrypoint did not finish SIGTERM handling" >&2 + exit 1 +fi +if ! wait "${entrypoint_pid}"; then + echo "Docker entrypoint did not exit cleanly after SIGTERM" >&2 + exit 1 +fi +if kill -0 "${child_pid}" 2>/dev/null; then + echo "Docker entrypoint left its Server child running" >&2 + exit 1 +fi + +rm -f "${TEST_HOME}/bin/pid" +( + cd "${TEST_HOME}" + exec setsid env \ + START_TEST_CHILD=true \ + START_TEST_DELAY=true \ + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +) & +entrypoint_pid=$! +for _ in $(seq 1 10); do + [[ -s "${TEST_HOME}/bin/pid" ]] && break + sleep 1 +done +[[ -s "${TEST_HOME}/bin/pid" ]] +child_pid=$(<"${TEST_HOME}/bin/pid") +kill -TERM "${entrypoint_pid}" +for _ in $(seq 1 15); do + ! kill -0 "${entrypoint_pid}" 2>/dev/null && break + sleep 1 +done +if kill -0 "${entrypoint_pid}" 2>/dev/null; then + kill -KILL -- "-${entrypoint_pid}" 2>/dev/null || true + wait "${entrypoint_pid}" 2>/dev/null || true + echo "Docker entrypoint did not handle SIGTERM during startup" >&2 + exit 1 +fi +if ! wait "${entrypoint_pid}"; then + echo "Docker entrypoint did not exit cleanly during startup" >&2 + exit 1 +fi +if kill -0 "${child_pid}" 2>/dev/null; then + echo "Docker entrypoint left its startup Server child running" >&2 + exit 1 +fi + +printf '%s\n' '-999999' > "${TEST_HOME}/bin/pid" +rm -f "${TEST_HOME}/docker/start-before-pid" +( + cd "${TEST_HOME}" + exec setsid env \ + START_TEST_PRE_PID_DELAY=true \ + HG_SERVER_BACKEND=hstore \ + HG_SERVER_ROCKSDB_PROVIDER=rocksdb \ + bash ./docker-entrypoint.sh +) & +entrypoint_pid=$! +for _ in $(seq 1 10); do + [[ -e "${TEST_HOME}/docker/start-before-pid" ]] && break + sleep 1 +done +[[ -e "${TEST_HOME}/docker/start-before-pid" ]] +[[ ! -e "${TEST_HOME}/bin/pid" ]] +kill -TERM "${entrypoint_pid}" +for _ in $(seq 1 15); do + ! kill -0 "${entrypoint_pid}" 2>/dev/null && break + sleep 1 +done +if kill -0 "${entrypoint_pid}" 2>/dev/null; then + kill -KILL -- "-${entrypoint_pid}" 2>/dev/null || true + wait "${entrypoint_pid}" 2>/dev/null || true + echo "Docker entrypoint did not ignore a stale pid during startup" >&2 + exit 1 +fi +if ! wait "${entrypoint_pid}"; then + echo "Docker entrypoint did not exit cleanly with a stale pid" >&2 + exit 1 +fi # The value start-hugegraph.sh actually received for -t, read from the # recorded argument vector so that -t 1200 can never satisfy an assertion # that wants 120. diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index b5ba2de34f..c764356bc8 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -19,8 +19,15 @@ set -euo pipefail DOCKER_FOLDER="./docker" INIT_FLAG_FILE="init_complete" -GRAPH_CONF="./conf/graphs/hugegraph.properties" REST_SERVER_CONF="./conf/rest-server.properties" +source ./bin/rocksdb-server-config.sh +SERVER_TOP="$(pwd -P)" +SERVER_GRAPHS_DIR=$(server_graphs_directory "$SERVER_TOP" "$REST_SERVER_CONF") +GRAPH_CONF="$SERVER_GRAPHS_DIR/hugegraph.properties" +[ -f "$GRAPH_CONF" ] || { + echo "Error: Docker requires hugegraph.properties in $SERVER_GRAPHS_DIR" >&2 + exit 1 +} mkdir -p "${DOCKER_FOLDER}" @@ -55,8 +62,25 @@ set_prop_encoded() { key_re="^[[:space:]]*${esc_key}([[:space:]]*[:=]|[[:space:]]+|[[:space:]]*$)" if grep -qE "${key_re}" "${file}"; then - sed -ri "0,/${key_re}/!{/${key_re}/d;}" "${file}" - sed -ri "0,/${key_re}/s~${key_re}.*~${key}=${esc_val}~" "${file}" + if sed --version >/dev/null 2>&1; then + # GNU sed supports the 0,/regexp/ address used by the Linux + # images. Keep the first property and remove later duplicates. + sed -ri "0,/${key_re}/!{/${key_re}/d;}" "${file}" + sed -ri "0,/${key_re}/s~${key_re}.*~${key}=${esc_val}~" "${file}" + else + # BSD sed (macOS) has neither -r nor the GNU line-0 address. Find + # the matching lines explicitly, delete duplicates from the end, + # then replace the first line in place. + first_line=$(grep -nE "${key_re}" "${file}" | head -n 1 | cut -d: -f1) + duplicate_lines=$(grep -nE "${key_re}" "${file}" | + cut -d: -f1 | tail -n +2 | sort -rn) + while IFS= read -r duplicate; do + [[ -z "${duplicate}" ]] || \ + sed -E -i '' "${duplicate}d" "${file}" + done <<< "${duplicate_lines}" + sed -E -i '' "${first_line}s~${key_re}.*~${key}=${esc_val}~" \ + "${file}" + fi else printf '%s=%s\n' "$key" "$encoded_val" >> "${file}" fi @@ -90,6 +114,66 @@ migrate_env() { migrate_env "BACKEND" "HG_SERVER_BACKEND" migrate_env "PD_PEERS" "HG_SERVER_PD_PEERS" +ROCKSDB_PROVIDER="${HG_SERVER_ROCKSDB_PROVIDER:-rocksdb}" +case "${ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) log "ERROR: HG_SERVER_ROCKSDB_PROVIDER must be rocksdb or topling" + exit 1 ;; +esac + +REQUESTED_BACKEND="${HG_SERVER_BACKEND:-$(get_prop_encoded "backend" "${GRAPH_CONF}")}" +REQUESTED_BACKEND=$(printf '%s' "$REQUESTED_BACKEND" | tr '[:upper:]' '[:lower:]') +if [[ "${REQUESTED_BACKEND}" == "hstore" ]]; then + TOPLING_JAR=$(find ./lib -path '*/topling/rocksdbjni*.jar' \ + -print -quit 2>/dev/null || true) + if [[ "${ROCKSDB_PROVIDER}" == "topling" ]]; then + log "ERROR: an HStore Server cannot select a local ToplingDB runtime" + exit 1 + fi + if [[ -n "${TOPLING_JAR}" || + -e ./library/librocksdbjni-linux64.so ]]; then + log "ERROR: an HStore Server image must not contain a local" \ + "ToplingDB runtime" + exit 1 + fi +fi + +if [[ -n "${HG_SERVER_DATA_PATH:-}" ]]; then + ROCKSDB_DATA_ROOT="${HG_SERVER_DATA_PATH}" +elif [[ "${ROCKSDB_PROVIDER}" == "topling" ]]; then + ROCKSDB_DATA_ROOT="$(pwd)/topling-data" +else + ROCKSDB_DATA_ROOT="$(pwd)/rocksdb-data" +fi +ENFORCE_PROVIDER_MARKER="${HG_SERVER_ENFORCE_PROVIDER_MARKER:-false}" +case "${ENFORCE_PROVIDER_MARKER}" in + true | false) ;; + *) log "ERROR: HG_SERVER_ENFORCE_PROVIDER_MARKER must be true or false" + exit 1 ;; +esac +VERIFIED_ROCKSDB_ROOT="" +if [[ "${REQUESTED_BACKEND}" == "rocksdb" || "${REQUESTED_BACKEND}" == "rocksdbsst" ]]; then + [[ "${ROCKSDB_DATA_ROOT}" == /* ]] || { + log "ERROR: RocksDB data path must be absolute: ${ROCKSDB_DATA_ROOT}" + exit 1 + } + ROCKSDB_DATA_ROOT=$(server_storage_path "$SERVER_TOP" "$ROCKSDB_DATA_ROOT") + [ -d "${ROCKSDB_DATA_ROOT}" ] || { + log "ERROR: RocksDB data path must be an existing directory: ${ROCKSDB_DATA_ROOT}" + exit 1 + } + # Keep the old root rejection points ahead of config rewrites. The locked + # provider claim follows the final graph mount preflight. + server_check_existing_markers "${ROCKSDB_DATA_ROOT}" "${ROCKSDB_PROVIDER}" + if [[ "${ROCKSDB_PROVIDER}" == topling || "${ENFORCE_PROVIDER_MARKER}" == true ]] && + [[ ! -f "${ROCKSDB_DATA_ROOT}/.hugegraph-rocksdb-provider" ]] && + find -H "${ROCKSDB_DATA_ROOT}" -mindepth 1 -maxdepth 1 \ + ! -name lost+found -print -quit | grep -q .; then + log "ERROR: refusing unmarked non-empty data path: ${ROCKSDB_DATA_ROOT}" + exit 1 + fi +fi + if [[ -n "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then LC_ALL=C if (( ${#HG_SERVER_AUTH_TOKEN_SECRET} < 32 )); then @@ -147,6 +231,11 @@ fi # ── Map env → properties file ───────────────────────────────────────── [[ -n "${HG_SERVER_BACKEND:-}" ]] && set_prop "backend" "${HG_SERVER_BACKEND}" "${GRAPH_CONF}" +set_prop "rocksdb.provider" "${ROCKSDB_PROVIDER}" "${GRAPH_CONF}" +if [[ "${REQUESTED_BACKEND}" == "rocksdb" || "${REQUESTED_BACKEND}" == "rocksdbsst" ]]; then + set_prop "rocksdb.data_path" "${ROCKSDB_DATA_ROOT}/data" "${GRAPH_CONF}" + set_prop "rocksdb.wal_path" "${ROCKSDB_DATA_ROOT}/wal" "${GRAPH_CONF}" +fi [[ -n "${HG_SERVER_PD_PEERS:-}" ]] && set_prop "pd.peers" "${HG_SERVER_PD_PEERS}" "${GRAPH_CONF}" [[ -n "${HG_SERVER_USE_PD:-}" ]] && \ set_prop "usePD" "${HG_SERVER_USE_PD}" "${REST_SERVER_CONF}" @@ -192,6 +281,27 @@ esac [[ -n "${INIT_STORE_ENABLED}" ]] && \ set_prop "init_store.enabled" "${INIT_STORE_ENABLED}" "${REST_SERVER_CONF}" +# Validate the final generated graph configuration before any database opens. +EFFECTIVE_PROVIDER=$(server_rocksdb_provider "$SERVER_GRAPHS_DIR") +server_check_provider_override "$EFFECTIVE_PROVIDER" +server_verify_db_mounts "$SERVER_TOP" "$SERVER_GRAPHS_DIR" +if [[ "${REQUESTED_BACKEND}" == "rocksdb" || "${REQUESTED_BACKEND}" == "rocksdbsst" ]]; then + ./bin/verify-rocksdb-provider.sh server "${ROCKSDB_PROVIDER}" \ + "${ROCKSDB_DATA_ROOT}" "${ENFORCE_PROVIDER_MARKER}" + VERIFIED_ROCKSDB_ROOT="${ROCKSDB_DATA_ROOT}" + LEGACY_INIT_MARKER="${DOCKER_FOLDER}/${INIT_FLAG_FILE}" + DOCKER_FOLDER="${ROCKSDB_DATA_ROOT}/.hugegraph-state" + mkdir -p "${DOCKER_FOLDER}" + if [[ "${ROCKSDB_PROVIDER}" == "rocksdb" && + -f "${LEGACY_INIT_MARKER}" && + ! -e "${DOCKER_FOLDER}/${INIT_FLAG_FILE}" ]]; then + cp "${LEGACY_INIT_MARKER}" "${DOCKER_FOLDER}/${INIT_FLAG_FILE}" + log "migrated the legacy RocksDB initialization marker" + fi +fi +server_verify_graph_roots "$SERVER_TOP" "$SERVER_GRAPHS_DIR" \ + "$EFFECTIVE_PROVIDER" "$ENFORCE_PROVIDER_MARKER" "$VERIFIED_ROCKSDB_ROOT" + # ── Build wait-storage env ───────────────────────────────────────────── WAIT_ENV=() [[ -n "${HG_SERVER_BACKEND:-}" ]] && WAIT_ENV+=("hugegraph.backend=${HG_SERVER_BACKEND}") @@ -240,7 +350,56 @@ else ./bin/init-store.sh fi -./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" +PID_FILE="./bin/pid" +START_PID="" +PID="" + +read_server_pid() { + local candidate + + candidate=$(cat "$PID_FILE" 2>/dev/null || true) + if [[ "$candidate" =~ ^[1-9][0-9]*$ ]]; then + printf '%s' "$candidate" + fi +} + +# A restarted container can retain the previous process id in its writable +# layer. Only trust a pid file created by the startup launched below. +rm -f "$PID_FILE" + +# shellcheck disable=SC2329 # Invoked by the TERM/INT trap below. +shutdown_server() { + local server_pid="${PID:-}" + + if [[ -z "$server_pid" ]]; then + server_pid=$(read_server_pid) + fi + if [[ -n "$server_pid" ]]; then + kill -TERM -- "$server_pid" 2>/dev/null || true + fi + if [[ -n "${START_PID:-}" ]]; then + kill -TERM -- "$START_PID" 2>/dev/null || true + fi + while [[ -n "$server_pid" ]] && kill -0 "$server_pid" 2>/dev/null; do + # kill -0 remains true for an unreaped zombie. Do not keep the + # container alive after the JVM has already completed shutdown. + if [[ -r "/proc/$server_pid/stat" ]]; then + PROCESS_STATE=$(awk '{ print $3 }' "/proc/$server_pid/stat" \ + 2>/dev/null || true) + if [[ "$PROCESS_STATE" == "Z" ]]; then + break + fi + fi + sleep 1 + done + exit 0 +} +trap shutdown_server TERM INT + +./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" & +START_PID=$! +wait "$START_PID" +START_PID="" # Post-startup cluster stabilization check (hstore only — rocksdb has no partitions) ACTUAL_BACKEND=$(grep -E '^[[:space:]]*backend[[:space:]]*=' "${GRAPH_CONF}" | head -n 1 | sed 's/.*=//' | tr -d '[:space:]' || true) @@ -250,9 +409,8 @@ if [[ "${ACTUAL_BACKEND}" == "hstore" ]]; then ./bin/wait-partition.sh || log "WARN: partitions not assigned yet" fi -PID=$(cat ./bin/pid 2>/dev/null || true) +PID=$(read_server_pid) if [[ -n "$PID" ]]; then - trap 'kill -TERM "$PID" 2>/dev/null; while kill -0 "$PID" 2>/dev/null; do sleep 1; done; exit 0' TERM INT tail --pid="$PID" -f /dev/null exit 1 fi diff --git a/hugegraph-server/hugegraph-dist/pom.xml b/hugegraph-server/hugegraph-dist/pom.xml index c4575bc877..5c88a931f8 100644 --- a/hugegraph-server/hugegraph-dist/pom.xml +++ b/hugegraph-server/hugegraph-dist/pom.xml @@ -112,6 +112,9 @@ ${basedir}/src/assembly/static true + + lib/topling/** + ${basedir}/src/main/resources diff --git a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml index b7f9ee0487..1b77565124 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml +++ b/hugegraph-server/hugegraph-dist/src/assembly/descriptor/assembly.xml @@ -37,6 +37,7 @@ false README.txt + lib/topling/** diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh new file mode 100644 index 0000000000..c9bdab237c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh @@ -0,0 +1,288 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -Eeuo pipefail +IFS=$'\n\t' +trap 'echo "[common-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + +GITHUB="https://github.com" + +function abs_path() { + local SOURCE + SOURCE="${BASH_SOURCE[0]}" + while [[ -h "$SOURCE" ]]; do + local DIR + DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + cd -P "$(dirname "$SOURCE")" && pwd +} + +function extract_so_with_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + + if [ ! -f "$jar_file" ]; then + echo "'$jar_file' Not Exist" >&2 + return 1 + fi + + mkdir -p "$dest_dir" || { + echo "Cannot mkdir '$dest_dir'" >&2 + return 1 + } + + if command -v realpath >/dev/null 2>&1; then + abs_jar_path="$(realpath "$jar_file")" + else + abs_jar_path="$(readlink -f "$jar_file")" + fi + if ! command -v unzip >/dev/null 2>&1; then + echo "Error: 'unzip' command not found. Please install unzip." >&2 + return 1 + fi + unzip -j -o "$abs_jar_path" "*.so" -d "$dest_dir" > /dev/null 2>&1 || { + local code=$? + if [ $code -eq 11 ]; then + echo "Error: No .so files found in '$abs_jar_path' (unzip exit 11)" >&2 + else + echo "Error: unzip failed (exit $code) for '$abs_jar_path'" >&2 + fi + return $code + } +} + +function extract_html_css_from_jar() { + local jar_file="$1" + local dest_dir="$2" + local abs_jar_path + local resource_target="$dest_dir/rocksdb_resource" + + if [ ! -f "$jar_file" ]; then + echo "Error: JAR file '$jar_file' does not exist." >&2 + return 1 + fi + + mkdir -p "$resource_target" || { + echo "Error: Cannot create resource directory '$resource_target'." >&2 + return 1 + } + + if command -v realpath >/dev/null 2>&1; then + abs_jar_path="$(realpath "$jar_file")" + else + abs_jar_path="$(readlink -f "$jar_file")" + fi + if ! command -v unzip >/dev/null 2>&1; then + echo "Error: 'unzip' command not found. Please install unzip." >&2 + return 1 + fi + unzip -j -o "$abs_jar_path" "*.html" "*.css" -d "$resource_target" > /dev/null || { + local code=$? + if [ $code -eq 11 ]; then + echo "Notice: No .html or .css files found in '$jar_file'." >&2 + return 0 + else + echo "Error: unzip failed with exit code $code" >&2 + return $code + fi + } + +} + +function ensure_libaio_symlink() { + local dest_dir="$1" + # Keep the Ubuntu 24.04 compatibility link inside the component runtime. + # Installation must never require sudo or modify /usr/lib. + if [ -f /etc/os-release ]; then + . /etc/os-release + if [ "${ID:-}" = "ubuntu" ] && + command -v dpkg >/dev/null 2>&1 && + dpkg --compare-versions "${VERSION_ID:-0}" "ge" "24.04" && + [ ! -e /usr/lib/x86_64-linux-gnu/libaio.so.1 ] && + [ -e /usr/lib/x86_64-linux-gnu/libaio.so.1t64 ]; then + mkdir -p "$dest_dir" + ln -sfn /usr/lib/x86_64-linux-gnu/libaio.so.1t64 \ + "$dest_dir/libaio.so.1" + echo "Prepared component-local libaio.so.1 compatibility link" + fi + fi +} + +function download_and_verify() { + local url=$1 + local filepath=$2 + local expected_sha256=$3 + local actual_sha256 + + if [[ -f $filepath ]]; then + echo "File $filepath exists. Verifying SHA-256 checksum..." + actual_sha256=$(sha256sum "$filepath" | awk '{ print $1 }') + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "SHA-256 checksum verification failed for $filepath. Expected: $expected_sha256, but got: $actual_sha256" + echo "Deleting $filepath..." + rm -f "$filepath" + else + echo "SHA-256 checksum verification succeeded for $filepath." + return 0 + fi + fi + + echo "Downloading $filepath..." + if ! curl -fL --retry 2 --retry-delay 2 --retry-max-time 90 \ + --connect-timeout 10 --max-time 30 -o "$filepath" "$url"; then + echo "Failed to download $filepath" >&2 + rm -f "$filepath" + return 1 + fi + + actual_sha256=$(sha256sum "$filepath" | awk '{ print $1 }') + if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "SHA-256 checksum verification failed for $filepath after download. Expected: $expected_sha256, but got: $actual_sha256" + rm -f "$filepath" + return 1 + fi + + return 0 +} + +function download_and_setup_jemalloc() { + local arch lib_file download_url expected_sha256 system_lib top + top=$1 + system_lib="" + + if [[ "${LD_PRELOAD:-}" == *"libjemalloc"* ]]; then + return 0 + fi + + # Prefer system-installed jemalloc if available + # Try ldconfig first to locate the shared object + if command -v ldconfig >/dev/null 2>&1; then + system_lib=$(ldconfig -p 2>/dev/null | awk '/jemalloc/{print $4}' | head -n1) + fi + # Fallback to common library paths if ldconfig is not available or found nothing + if [[ -z "$system_lib" ]]; then + for p in \ + /usr/lib/libjemalloc.so \ + /usr/lib/libjemalloc.so.2 \ + /usr/lib64/libjemalloc.so \ + /usr/lib64/libjemalloc.so.2 \ + /usr/local/lib/libjemalloc.so \ + /usr/local/lib/libjemalloc.so.2 \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so \ + /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so \ + /usr/lib/aarch64-linux-gnu/libjemalloc.so.2; do + if [[ -f "$p" ]]; then + system_lib="$p" + break + fi + done + fi + + # If found, set LD_PRELOAD and return immediately + if [[ -n "$system_lib" ]]; then + export LD_PRELOAD="${system_lib}${LD_PRELOAD:+:$LD_PRELOAD}" + return 0 + fi + + # Detect system architecture + arch=$(uname -m) + + # System jemalloc not found, try to download the correct library for the architecture + # Checksums match apache/hugegraph-doc@567625c6ec66907fc60f1864146fbec91b5f6204. + if [[ $arch == "aarch64" || $arch == "arm64" ]]; then + lib_file="$top/bin/libjemalloc_aarch64.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc_aarch64.so" + expected_sha256="6b7e6099b6da798829c6ce6fcb55a787508841edd52446332a73300889dcd1dc" + elif [[ $arch == "x86_64" ]]; then + lib_file="$top/bin/libjemalloc.so" + download_url="${GITHUB}/apache/hugegraph-doc/raw/binary-1.5/dist/server/libjemalloc.so" + expected_sha256="53b25e8626e1605cbd8b60befb3431cabc1b8851a54285e0dda412796feab67d" + else + echo "Unsupported architecture: $arch" + return 1 + fi + + # Download and verify jemalloc library (fallback when system lib not found) + if download_and_verify "$download_url" "$lib_file" "$expected_sha256"; then + export LD_PRELOAD="${lib_file}${LD_PRELOAD:+:$LD_PRELOAD}" + else + echo "Failed to verify or download jemalloc for $arch, skipping" + return 1 + fi +} + +function require_topling_platform() { + local os_name machine_arch + + os_name="$(uname -s)" + machine_arch="$(uname -m)" + if [ "$os_name" != "Linux" ] || + [[ "$machine_arch" != "x86_64" ]]; then + printf 'Error: ToplingDB native runtime supports Linux x86_64 only; ' >&2 + printf 'current platform is %s/%s\n' "$os_name" "$machine_arch" >&2 + return 1 + fi +} + +function prepare_toplingdb() { + local lib_dir="$1" + local dest_dir="$2" + local top_override="${3:-}" + + require_topling_platform || return 1 + + local top + if [ -n "$top_override" ]; then + top="$top_override" + else + top="$(cd "$lib_dir"/../ && pwd)" || { + echo "Error: failed to resolve the ToplingDB installation directory" >&2 + return 1 + } + fi + + local jar_file + jar_file=$(ls -1 "$lib_dir"/rocksdbjni*.jar 2>/dev/null | sort -V | tail -n1 || true) + if [ -z "${jar_file:-}" ]; then + echo "Error: No rocksdbjni*.jar found under '$lib_dir'" >&2 + return 1 + fi + + ensure_libaio_symlink "$dest_dir" + if ! download_and_setup_jemalloc "$top"; then + echo "Warning: jemalloc is unavailable; continuing without it" >&2 + fi + extract_so_with_jar "$jar_file" "$dest_dir" + if ! extract_html_css_from_jar "$jar_file" "$dest_dir"; then + echo "Warning: failed to extract optional ToplingDB web resources; continuing" >&2 + fi + if [ -d "$dest_dir" ]; then + if [[ ":${LD_LIBRARY_PATH:-}:" != *":$dest_dir:"* ]]; then + export LD_LIBRARY_PATH="$dest_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + fi + + if [ -f "$dest_dir/librocksdbjni-linux64.so" ] && [[ ":${LD_PRELOAD:-}:" != *"librocksdbjni-linux64.so:"* ]]; then + export LD_PRELOAD="${LD_PRELOAD:+$LD_PRELOAD:}$dest_dir/librocksdbjni-linux64.so" + fi + else + echo "Warn: LD paths skipped, directory '$dest_dir' does not exist." >&2 + fi +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh index 14b6ac1f79..0d7ada9276 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/dump-store.sh @@ -51,5 +51,5 @@ echo "Dumping HugeGraph Store($conf)..." dump_store_ext_jar_path=$LIB/hugegraph-dist-*.jar for i in $LIB/*.jar; do dump_store_ext_jar_path=$dump_store_ext_jar_path:$i; export dump_store_ext_jar_path; done -exec "$JAVA" -cp "$dump_store_ext_jar_path" \ +exec "$JAVA" "-Dlog4j.configurationFile=${CONF}/log4j2.xml" -cp "$dump_store_ext_jar_path" \ org.apache.hugegraph.cmd.StoreDumper "$@" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index caffedc482..3e3e3db42c 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -70,17 +70,18 @@ MIN_JAVA_VERSION=11 MAX_SECURITY_JAVA_VERSION=23 # Add the slf4j-log4j12 binding -CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') +CP=$(find_standard_lib_jars "$LIB" 'log4j-slf4j-impl*.jar' | + sort | tr '\n' ':') # Add the jars in lib that start with "hugegraph" -CP="$CP":$(find -L $LIB -name 'hugegraph*.jar' | sort | tr '\n' ':') +CP="$CP":$(find_standard_lib_jars "$LIB" 'hugegraph*.jar' | + sort | tr '\n' ':') # Add the remaining jars in lib. -CP="$CP":$(find -L $LIB -name '*.jar' \ - \! -name 'hugegraph*' \ - \! -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') +CP="$CP":$(find_standard_lib_jars "$LIB" '*.jar' \ + 'hugegraph*' 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') # Add the jars in ext (at any subdirectory depth) -CP="$CP":$(find -L $EXT -name '*.jar' | sort | tr '\n' ':') +CP="$CP":$(find -L "$EXT" -name '*.jar' | sort | tr '\n' ':') # Add the jars in plugins (at any subdirectory depth), check "javaagent" related jars carefully -CP="$CP":$(find -L $PLUGINS -name '*.jar' | sort | tr '\n' ':') +CP="$CP":$(find -L "$PLUGINS" -name '*.jar' | sort | tr '\n' ':') # (Cygwin only) Use ; classpath separator and reformat paths for Windows ("C:\foo") [[ $(uname) = CYGWIN* ]] && CP="$(cygpath -p -w "$CP")" @@ -90,6 +91,8 @@ export CLASSPATH="${CLASSPATH:-}:$CP" # Change to $BIN's parent cd "${TOP}" || exit 1 +source "$BIN/preload-topling.sh" + # Find java & enable server option if [ "$JAVA_HOME" = "" ]; then JAVA="java -server" @@ -259,13 +262,13 @@ fi # Turn on security check if [[ "${STDOUT_MODE:-false}" == "true" ]]; then exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ - ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ + ${SECURITY_MANAGER_OPTION} -cp "${CLASSPATH}:" \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ - ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} + "${OPEN_SECURITY_CHECK}" "${GREMLIN_SERVER_CONF}" "${REST_SERVER_CONF}" else exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ - ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ + ${SECURITY_MANAGER_OPTION} -cp "${CLASSPATH}:" \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ - ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} \ + "${OPEN_SECURITY_CHECK}" "${GREMLIN_SERVER_CONF}" "${REST_SERVER_CONF}" \ >> ${LOGS}/hugegraph-server-stdout.log 2>&1 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh index 74ec0bb731..7fd804251f 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh @@ -48,13 +48,19 @@ cd "${TOP}" || exit DEFAULT_JAVA_OPTIONS="--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" +source "$BIN/preload-topling.sh" + echo "Initializing HugeGraph Store..." # Build classpath with hugegraph*.jar first to avoid class loading conflicts -CP=$(find -L "${LIB}" -name 'hugegraph*.jar' | sort | tr '\n' ':') -CP="$CP":$(find -L "${LIB}" -name '*.jar' \! -name 'hugegraph*' | sort | tr '\n' ':') +CP=$(find_standard_lib_jars "${LIB}" 'hugegraph*.jar' | sort | tr '\n' ':') +if [ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]; then + CP="$TOPLING_RUNTIME_CLASSPATH:$CP" +fi +CP="$CP":$(find_standard_lib_jars "${LIB}" '*.jar' 'hugegraph*' | + sort | tr '\n' ':') CP="$CP":$(find -L "${PLUGINS}" -name '*.jar' | sort | tr '\n' ':') -$JAVA -cp $CP ${DEFAULT_JAVA_OPTIONS} \ +$JAVA -cp "$CP" ${DEFAULT_JAVA_OPTIONS} \ org.apache.hugegraph.cmd.InitStore "${CONF}"/rest-server.properties INIT_STORE_STATUS=$? if [[ ${INIT_STORE_STATUS} -ne 0 ]]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh new file mode 100644 index 0000000000..c41982675b --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh @@ -0,0 +1,230 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ORIG_SHELL_FLAGS="$-" +ORIG_PIPEFAIL="$(set -o | awk '$1 == "pipefail" { print $2 }')" +ORIG_ERR_TRAP="$(trap -p ERR)" +# Save original IFS to avoid leaking into parent shell when sourced +ORIG_IFS="${IFS}" +set -Eeuo pipefail +IFS=$'\n\t' +# Unified error capture for easy positioning +trap 'echo "[preload-topling] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + +RUNTIME_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPONENT_TOP="$(cd "$RUNTIME_BIN"/../ && pwd)" +COMPONENT_LIB="$COMPONENT_TOP/lib" +DEST_DIR="$COMPONENT_TOP/library" + +detect_rocksdb_provider() { + local conf_dir="$1" + local file value provider="" seen_provider=false + + while IFS= read -r value; do + if [ "$seen_provider" = false ]; then + provider="$value" + seen_provider=true + elif [ "$provider" != "$value" ]; then + echo "Error: conflicting rocksdb.provider values: $provider,$value" >&2 + return 1 + fi + done < <( + for file in "$conf_dir"/graphs/*.properties; do + [ -f "$file" ] || continue + awk ' + /^[[:space:]]*#/ { next } + /^[[:space:]]*rocksdb\.provider[[:space:]]*=/ { + value = $0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + print value + } + ' "$file" + done + for file in "$conf_dir"/application*.yml; do + [ -f "$file" ] || continue + awk ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + /^[^[:space:]#][^:]*:/ { + in_rocksdb = ($0 ~ /^rocksdb[[:space:]]*:/) + } + in_rocksdb && /^[[:space:]]+provider[[:space:]]*:/ { + value = $0 + sub(/^[^:]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (value ~ /^"[^"]*"$/ || value ~ /^'\''[^'\'']*'\''$/) { + value = substr(value, 2, length(value) - 2) + } + print value + } + ' "$file" + done + ) + + provider="${provider:-rocksdb}" + case "$provider" in + rocksdb | topling) + echo "$provider" + ;; + *) + echo "Error: invalid rocksdb.provider '$provider'; expected rocksdb or topling" >&2 + return 1 + ;; + esac +} + +SERVER_CONFIG_ACTIVE=false +if [ -f "$RUNTIME_BIN/rocksdb-server-config.sh" ] || + [ -f "${REST_SERVER_CONF:-$COMPONENT_TOP/conf/rest-server.properties}" ] || + [ -d "$COMPONENT_TOP/conf/graphs" ]; then + SERVER_CONFIG_ACTIVE=true + source "$RUNTIME_BIN/rocksdb-server-config.sh" + SERVER_REST_CONFIG="${REST_SERVER_CONF:-$COMPONENT_TOP/conf/rest-server.properties}" + [[ "$SERVER_REST_CONFIG" = /* ]] || SERVER_REST_CONFIG="$COMPONENT_TOP/$SERVER_REST_CONFIG" + SERVER_GRAPHS_DIR=$(server_graphs_directory "$COMPONENT_TOP" "$SERVER_REST_CONFIG") || exit 1 + PROVIDER=$(server_rocksdb_provider "$SERVER_GRAPHS_DIR") || exit 1 + server_check_provider_override "$PROVIDER" || exit 1 +else + PROVIDER="${TOPLINGDB_ROCKSDB_PROVIDER:-}" + if [ -n "$PROVIDER" ]; then + case "$PROVIDER" in + rocksdb | topling) ;; + *) + echo "Error: invalid TOPLINGDB_ROCKSDB_PROVIDER '$PROVIDER';" \ + "expected rocksdb or topling" >&2 + exit 1 + ;; + esac + else + PROVIDER=$(detect_rocksdb_provider "$COMPONENT_TOP/conf") || exit 1 + fi +fi + +remove_path_entry() { + local value="${1:-}" + local remove="${2:-}" + local entry result="" + local path_ifs="$IFS" + IFS=: + for entry in $value; do + [ -n "$entry" ] && [ "$entry" != "$remove" ] || continue + result="${result:+$result:}$entry" + done + IFS="$path_ifs" + echo "$result" +} + +# A parent launcher may start multiple components from one shell. Remove only +# the runtime entry previously selected by this helper before selecting ours. +if [ -n "${TOPLING_ACTIVE_NATIVE:-}" ]; then + LD_PRELOAD=$(remove_path_entry "${LD_PRELOAD:-}" "$TOPLING_ACTIVE_NATIVE") + LD_LIBRARY_PATH=$(remove_path_entry "${LD_LIBRARY_PATH:-}" \ + "$(dirname "$TOPLING_ACTIVE_NATIVE")") + export LD_PRELOAD LD_LIBRARY_PATH +fi +if [ -n "${TOPLING_ACTIVE_JAR:-}" ]; then + CLASSPATH=$(remove_path_entry "${CLASSPATH:-}" "$TOPLING_ACTIVE_JAR") + export CLASSPATH +fi +unset TOPLING_ACTIVE_NATIVE TOPLING_ACTIVE_JAR TOPLING_RUNTIME_CLASSPATH + +if [ "$PROVIDER" = "topling" ]; then + # Runtime binaries are read-only; installation prepares them beforehand. + if [ "$(uname -s)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then + echo "Error: ToplingDB runtime supports Linux x86_64 only" >&2 + exit 1 + fi + if [ "$SERVER_CONFIG_ACTIVE" = true ]; then + server_verify_graph_roots "$COMPONENT_TOP" "$SERVER_GRAPHS_DIR" "$PROVIDER" \ + "${HG_SERVER_ENFORCE_PROVIDER_MARKER:-false}" || exit 1 + fi + TOPLING_JAR=$(ls -1 "$COMPONENT_LIB"/topling/rocksdbjni*.jar 2>/dev/null | + sort -V | tail -1 || true) + if [ -z "$TOPLING_JAR" ]; then + echo "Error: no prepared ToplingDB JAR found in $COMPONENT_LIB/topling/" >&2 + exit 1 + fi + + CONF_FILE="${TOPLINGDB_EASY_MIGRATE_CONF:-}" + if [ -z "$CONF_FILE" ]; then + CONF_FILE="$COMPONENT_TOP/conf/toplingdb.yaml" + if [ ! -f "$CONF_FILE" ]; then + CONF_FILE="$COMPONENT_TOP/conf/rocksdb_store.yaml" + fi + if [ ! -f "$CONF_FILE" ]; then + CONF_FILE="$COMPONENT_TOP/conf/rocksdb_pd.yaml" + fi + fi + if [ ! -f "$CONF_FILE" ]; then + echo "Error: required ToplingDB Easy Migrate config not found: $CONF_FILE" >&2 + exit 1 + fi + if [ ! -r "$CONF_FILE" ]; then + echo "Error: ToplingDB Easy Migrate config is not readable: $CONF_FILE" >&2 + exit 1 + fi + export TOPLINGDB_EASY_MIGRATE_CONF="$CONF_FILE" + echo "[preload-topling] TOPLINGDB_EASY_MIGRATE_CONF=$CONF_FILE" + NATIVE_LIBRARY="$DEST_DIR/librocksdbjni-linux64.so" + if [ ! -r "$NATIVE_LIBRARY" ]; then + echo "Error: prepared ToplingDB native library not found: $NATIVE_LIBRARY" >&2 + echo " Run bin/prepare-topling.sh for this component before startup." >&2 + exit 1 + fi + export LD_LIBRARY_PATH="$DEST_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + if command -v ldd >/dev/null 2>&1 && + ldd "$NATIVE_LIBRARY" 2>/dev/null | grep -q 'not found'; then + echo "Error: ToplingDB native library has unresolved system dependencies" >&2 + echo " Install them during package/deployment preparation." >&2 + exit 1 + fi + + export LD_PRELOAD="${LD_PRELOAD:+$LD_PRELOAD:}$NATIVE_LIBRARY" + export TOPLING_ACTIVE_NATIVE="$NATIVE_LIBRARY" + export TOPLING_RUNTIME_CLASSPATH="$TOPLING_JAR" + export CLASSPATH="$TOPLING_JAR${CLASSPATH:+:$CLASSPATH}" + export TOPLING_ACTIVE_JAR="$TOPLING_JAR" +else + if [ "$SERVER_CONFIG_ACTIVE" = true ]; then + server_verify_graph_roots "$COMPONENT_TOP" "$SERVER_GRAPHS_DIR" "$PROVIDER" \ + "${HG_SERVER_ENFORCE_PROVIDER_MARKER:-false}" || exit 1 + fi + unset TOPLINGDB_EASY_MIGRATE_CONF + echo "[preload-topling] Component uses rocksdb provider" +fi + +unset -f detect_rocksdb_provider remove_path_entry + +# Restore original IFS +IFS="$ORIG_IFS" +if [ -n "$ORIG_ERR_TRAP" ]; then + eval "$ORIG_ERR_TRAP" +else + trap - ERR +fi +# Restore shell options to their state before this script was sourced +case "$ORIG_SHELL_FLAGS" in *e*) set -e ;; *) set +e ;; esac +case "$ORIG_SHELL_FLAGS" in *u*) set -u ;; *) set +u ;; esac +case "$ORIG_SHELL_FLAGS" in *E*) set -E ;; *) set +E ;; esac +if [ "$ORIG_PIPEFAIL" = "on" ]; then + set -o pipefail +else + set +o pipefail +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh new file mode 100755 index 0000000000..fcfe243e7c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOP="$(cd "$BIN"/.. && pwd)" +TOPLING_LIB="$TOP/lib/topling" +RUNTIME_DIR="$TOP/library" + +if [ ! -r "$BIN/common-topling.sh" ]; then + echo "Error: ToplingDB preparation helper not found: $BIN/common-topling.sh" >&2 + exit 1 +fi + +source "$BIN/common-topling.sh" +prepare_toplingdb "$TOPLING_LIB" "$RUNTIME_DIR" "$TOP" + +echo "[prepare-topling] Runtime prepared under $RUNTIME_DIR" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/rocksdb-server-config.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/rocksdb-server-config.sh new file mode 100644 index 0000000000..62dc344f72 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/rocksdb-server-config.sh @@ -0,0 +1,352 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Server properties are authoritative for both Java and native runtime selection. +# Deliberately reject ambiguous syntax instead of approximating HugeConfig. +server_property() { + local file="$1" key="$2" fallback="${3-}" + [ -r "$file" ] || { echo "Error: unreadable configuration: $file" >&2; return 1; } + awk -v wanted="$key" -v fallback="$fallback" ' + function fail(reason) { + print "Error: " reason " for " wanted " in " FILENAME > "/dev/stderr" + failed = 1 + exit 1 + } + /^[[:space:]]*[#!]/ || /^[[:space:]]*$/ { next } + { + line = $0 + sub(/^[[:space:]]+/, "", line) + key = line + sub(/[[:space:]:=].*$/, "", key) + if (key ~ /\\/) fail("escaped property keys are unsupported") + if (key == "include" || key == "includeOptional") fail("property includes are unsupported") + if (line ~ /\\$/) fail("property continuations are unsupported") + if (key != wanted) next + if (seen++) fail("duplicate property") + value = substr(line, length(key) + 1) + sub(/^[[:space:]]*/, "", value) + sub(/^[:=][[:space:]]*/, "", value) + sub(/[[:space:]]+$/, "", value) + if (value ~ /\$\{/ || value ~ /,/) fail("interpolation/list syntax is unsupported") + decoded = "" + for (i = 1; i <= length(value); i++) { + c = substr(value, i, 1) + if (c == "\\") { + c = substr(value, ++i, 1) + if (c != " " && c != "\\" && c != ":" && c != "=" && c != "#" && c != "!") { + fail("unsupported escape or continuation") + } + } + decoded = decoded c + } + } + END { + if (!failed) print seen ? decoded : fallback + } + ' "$file" +} + +server_graphs_directory() { + local top="$1" rest="$2" graphs + graphs=$(server_property "$rest" graphs './conf/graphs') || return 1 + [ -n "$graphs" ] || { echo "Error: empty graphs directory" >&2; return 1; } + [[ "$graphs" = /* ]] || graphs="$top/$graphs" + [ -d "$graphs" ] || { echo "Error: graphs directory does not exist: $graphs" >&2; return 1; } + printf '%s\n' "$graphs" +} + +server_rocksdb_provider() { + local graphs="$1" file backend value provider="" + for file in "$graphs"/*.properties; do + [ -f "$file" ] || continue + backend=$(server_property "$file" backend memory) || return 1 + backend=$(printf '%s' "$backend" | tr '[:upper:]' '[:lower:]') + # Ignore remote/in-memory owners; a provider on them cannot load JNI. + case "$backend" in rocksdb | rocksdbsst) ;; *) continue ;; esac + value=$(server_property "$file" rocksdb.provider rocksdb) || return 1 + case "$value" in + rocksdb | topling) ;; + *) echo "Error: invalid rocksdb.provider '$value' in $file" >&2; return 1 ;; + esac + if [ -n "$provider" ] && [ "$provider" != "$value" ]; then + echo "Error: conflicting rocksdb.provider values: $provider,$value" >&2 + return 1 + fi + provider="$value" + done + printf '%s\n' "${provider:-rocksdb}" +} + +server_check_provider_override() { + local provider="$1" override="${TOPLINGDB_ROCKSDB_PROVIDER:-}" + case "$override" in + '' | rocksdb | topling) ;; + *) echo "Error: invalid TOPLINGDB_ROCKSDB_PROVIDER '$override'" >&2; return 1 ;; + esac + if [ -n "$override" ] && [ "$override" != "$provider" ]; then + echo "Error: TOPLINGDB_ROCKSDB_PROVIDER=$override conflicts with configured rocksdb.provider=$provider" >&2 + return 1 + fi +} + +# Resolve storage paths without GNU realpath, including missing final directories. +# Canonicalizing only the distribution prefix permits macOS /var aliases while +# still rejecting a symlink configured below that prefix or in an external root. +server_lexical_path() { + printf '%s\n' "$1" | awk ' + { + count = split($0, parts, "/") + depth = 0 + for (i = 1; i <= count; i++) { + if (parts[i] == "" || parts[i] == ".") continue + if (parts[i] == "..") { + if (depth > 0) depth-- + } else { + stack[++depth] = parts[i] + } + } + result = "" + for (i = 1; i <= depth; i++) result = result "/" stack[i] + print (result == "" ? "/" : result) + } + ' +} + +server_storage_path() { + local top="$1" path="$2" physical_top lexical physical probe suffix="" + local part prefix="" + local -a parts + physical_top=$(cd -P "$top" && pwd) || return 1 + if [[ "$path" != /* ]]; then + path="$physical_top/$path" + else + case "$path" in "$top" | "$top/"*) path="$physical_top${path#"$top"}" ;; esac + fi + IFS=/ read -r -a parts <<< "$path" + for part in "${parts[@]}"; do + [ -n "$part" ] || continue + prefix="$prefix/$part" + if [ -L "$prefix" ]; then + echo "Error: RocksDB storage path contains a symlink: $path" >&2 + return 1 + fi + done + lexical=$(server_lexical_path "$path") || return 1 + probe="$path" + while [ "$probe" != / ] && [[ "$probe" = */ ]]; do probe="${probe%/}"; done + while [ ! -d "$probe" ]; do + if [ -e "$probe" ] || [ -L "$probe" ]; then + echo "Error: RocksDB storage path is not a directory: $probe" >&2 + return 1 + fi + suffix="/${probe##*/}$suffix" + probe=$(dirname "$probe") + done + physical=$(cd -P "$probe" && pwd) || return 1 + physical=$(server_lexical_path "$physical$suffix") || return 1 + [ "$lexical" = "$physical" ] || { + echo "Error: RocksDB storage path contains a symlink: $path" >&2 + return 1 + } + printf '%s\n' "$lexical" +} + +# Standard RocksDB also runs on macOS. Inspect ownership without Linux flock/proc. +server_check_existing_markers() { + local path="$1" provider="$2" current physical expected actual marker + expected=$(printf '%s\n' format=1 component=server "provider=$provider") + physical="$path" + while [ ! -d "$physical" ] && [ "$physical" != / ]; do + if [ -e "$physical" ] || [ -L "$physical" ]; then + echo "Error: RocksDB storage path is not a directory: $physical" >&2 + return 1 + fi + physical=$(dirname "$physical") + done + physical=$(cd -P "$physical" && pwd) || return 1 + # Inspect both the configured ancestry and resolved destination ancestry. + # This catches a symlink into a differently marked deployment directory. + for current in "$path" "$physical"; do + while [ -n "$current" ]; do + marker="$current/.hugegraph-rocksdb-provider" + if [ -L "$marker" ] || { [ -e "$marker" ] && [ ! -f "$marker" ]; }; then + echo "Error: provider marker is not a regular file: $marker" >&2 + return 1 + fi + if [ ! -e "$marker" ] && [ -d "$current" ] && + find "$current" -mindepth 1 -maxdepth 1 -name '.provider-marker.*' -print -quit | grep -q .; then + echo "Error: provider marker initialization is in progress in $current" >&2 + return 1 + fi + if [ -f "$marker" ]; then + [ -r "$marker" ] || { echo "Error: provider marker is not readable: $marker" >&2; return 1; } + actual=$(cat "$marker") || return 1 + if [ "$actual" != "$expected" ]; then + echo "Error: provider marker mismatch in $current; expected server/$provider" >&2 + return 1 + fi + fi + [ "$current" != / ] || break + current=$(dirname "$current") + done + done +} + +# Nonempty legacy standard roots remain readable without a migration. New or +# empty roots must be claimed before opening: otherwise a concurrent Topling +# startup could claim them between the legacy check and the first data write. +server_claim_standard_path() { + local path="$1" current marker="$1/.hugegraph-rocksdb-provider" + server_check_existing_markers "$path" rocksdb || return 1 + current="$path" + while [ -n "$current" ]; do + # A verified deployment ancestor already owns its descendants. + [ ! -f "$current/.hugegraph-rocksdb-provider" ] || return 0 + [ "$current" != / ] || break + current=$(dirname "$current") + done + mkdir -p "$path" || return 1 + # Check again after creating missing directories, before changing ownership. + server_check_existing_markers "$path" rocksdb || return 1 + if find "$path" -mindepth 1 -maxdepth 1 ! -name lost+found \ + ! -name '.provider-marker.*' ! -name '.hugegraph-rocksdb-provider' -print -quit | grep -q .; then + server_check_existing_markers "$path" rocksdb + return $? + fi + # Bash noclobber atomically refuses an existing marker. Never replace the + # winner, including an interrupted/partial marker: those fail validation. + if ! (umask 077; set -C; printf '%s\n' format=1 component=server provider=rocksdb > "$marker") 2>/dev/null; then + [ -f "$marker" ] || { echo "Error: could not claim provider marker: $marker" >&2; return 1; } + fi + server_check_existing_markers "$path" rocksdb || return 1 + [ -f "$marker" ] || { echo "Error: provider marker disappeared: $marker" >&2; return 1; } +} + +# A configured data/WAL directory may sit below a deployment-owned marked root. +# Check every marked ancestor too, so a nested directory cannot bypass a marker. +server_verify_storage_path() { + local top="$1" provider="$2" path="$3" enforce="$4" + local current marked=false verified_root="${5:-}" + path=$(server_storage_path "$top" "$path") || return 1 + [ "$path" != / ] || { echo "Error: RocksDB data path cannot be /" >&2; return 1; } + if [ "$provider" = rocksdb ] && [ "$enforce" = false ]; then + server_claim_standard_path "$path" + return $? + fi + current="${path%/}" + while [ -n "$current" ] && [ "$current" != / ]; do + if [ -e "$current/.hugegraph-rocksdb-provider" ] || + [ -L "$current/.hugegraph-rocksdb-provider" ]; then + "$top/bin/verify-rocksdb-provider.sh" server "$provider" "$current" "$enforce" || return 1 + marked=true + fi + current="${current%/*}" + done + # The Docker-generated root was already checked, including legacy unmarked + # RocksDB mode. Its data/WAL children need not exist before first init. + if [ -n "$verified_root" ]; then + verified_root=$(server_storage_path "$top" "$verified_root") || return 1 + case "$path/" in "$verified_root/"*) marked=true ;; esac + fi + if [ "$marked" = false ]; then + "$top/bin/verify-rocksdb-provider.sh" server "$provider" "$path" "$enforce" || return 1 + fi +} + +# Check every local DB before any graph root is claimed or a JVM starts. The +# native recovery lock rejects a bind-mounted DB, but g/m/s open concurrently: +# discovering the bad mount there lets the other stores initialize first. +server_verify_db_mounts() { + local top="$1" graphs="$2" file backend path db_path store status system + local version major minor + local checked=false + system=$(uname -s) || return 1 + [ "$system" = Linux ] || return 0 + for file in "$graphs"/*.properties; do + [ -f "$file" ] || continue + backend=$(server_property "$file" backend memory) || return 1 + backend=$(printf '%s' "$backend" | tr '[:upper:]' '[:lower:]') + case "$backend" in rocksdb | rocksdbsst) ;; *) continue ;; esac + if [ "$checked" = false ]; then + [ -r /proc/self/mountinfo ] || { + echo "Error: Linux mount table is required to validate RocksDB directories" >&2 + return 1 + } + command -v mountpoint >/dev/null 2>&1 || { + echo "Error: util-linux mountpoint is required to validate RocksDB directories" >&2 + return 1 + } + version=$(LC_ALL=C mountpoint --version 2>/dev/null) || return 1 + if [[ ! "$version" =~ ^mountpoint[[:space:]]from[[:space:]]util-linux[[:space:]]([0-9]+)\.([0-9]+) ]]; then + echo "Error: util-linux mountpoint is required to validate RocksDB directories" >&2 + return 1 + fi + major="${BASH_REMATCH[1]}" + minor="${BASH_REMATCH[2]}" + if (( major < 2 || (major == 2 && minor < 37) )); then + echo "Error: util-linux mountpoint 2.37+ is required to validate RocksDB directories" >&2 + return 1 + fi + checked=true + fi + path=$(server_property "$file" rocksdb.data_path 'rocksdb-data/data') || return 1 + [ -n "$path" ] || { echo "Error: empty rocksdb.data_path in $file" >&2; return 1; } + path=$(server_storage_path "$top" "$path") || return 1 + for store in m g s; do + db_path="$path/$store" + [ -d "$db_path" ] || continue + if mountpoint -q -- "$db_path"; then + echo "Error: RocksDB directory cannot itself be a mount point;" \ + "mount its parent data root instead: $db_path" >&2 + return 1 + else + status=$? + [ "$status" -eq 32 ] || { + echo "Error: failed to inspect RocksDB mount point: $db_path" >&2 + return 1 + } + fi + done + done +} + +server_verify_graph_roots() { + local top="$1" graphs="$2" provider="$3" enforce="$4" + local file backend disks path key verified_root="${5:-}" + case "$enforce" in + true | false) ;; + *) echo "Error: provider marker enforcement must be true or false" >&2; return 1 ;; + esac + server_verify_db_mounts "$top" "$graphs" || return 1 + for file in "$graphs"/*.properties; do + [ -f "$file" ] || continue + backend=$(server_property "$file" backend memory) || return 1 + backend=$(printf '%s' "$backend" | tr '[:upper:]' '[:lower:]') + case "$backend" in rocksdb | rocksdbsst) ;; *) continue ;; esac + disks=$(server_property "$file" rocksdb.data_disks '') || return 1 + [ -z "$disks" ] || { + echo "Error: provider marker validation does not support rocksdb.data_disks in $file" >&2 + return 1 + } + for key in data wal; do + path=$(server_property "$file" "rocksdb.${key}_path" "rocksdb-data/$key") || return 1 + [ -n "$path" ] || { echo "Error: empty rocksdb.${key}_path in $file" >&2; return 1; } + server_verify_storage_path "$top" "$provider" "$path" "$enforce" "$verified_root" || return 1 + done + done +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 2c99238327..56986ee5a9 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -99,6 +99,8 @@ if [[ $PRELOAD == "true" ]]; then sed -i -e '/registerBackends/d; /serverStarted/d' "${SCRIPTS}/${EXAMPLE_SCRIPT}" fi +source "$BIN/preload-topling.sh" + if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphServer in daemon mode..." "${BIN}"/hugegraph-server.sh "${CONF}/${GREMLIN_SERVER_CONF}" "${CONF}"/rest-server.properties \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 8ae46e88a4..99744a88af 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -24,6 +24,75 @@ function command_available() { return 1 } +# Resolve a path before comparing it with an excluded runtime directory. The +# fallback keeps the launcher usable on macOS hosts where `realpath` may not be +# installed, while the depth limit prevents malformed symlink loops from +# hanging startup. +function canonical_path() { + local path="$1" + local depth="${2:-0}" + local target + local dir + + (( depth < 40 )) || return 1 + if command_available "realpath"; then + realpath "$path" + return + fi + if [[ -L "$path" ]]; then + target=$(readlink "$path") || return 1 + if [[ "$target" = /* ]]; then + canonical_path "$target" "$((depth + 1))" + else + canonical_path "$(dirname "$path")/$target" "$((depth + 1))" + fi + return + fi + dir=$(cd -P "$(dirname "$path")" 2>/dev/null && pwd) || return 1 + printf '%s/%s\n' "$dir" "$(basename "$path")" +} + +# Print JAR paths below a component's lib directory except anything whose +# canonical target lives below its optional Topling subtree. `find -P` keeps +# directory aliases from being traversed, while direct file aliases are +# resolved and emitted canonically after the exclusion check. +function find_standard_lib_jars() { + local lib_dir="$1" + local pattern="$2" + local -a excluded_patterns=("${@:3}") + local top_dir="$lib_dir/topling" + local canonical_top + local jar + local canonical_jar + local name + local excluded + + if [[ -d "$top_dir" ]]; then + canonical_top=$(canonical_path "$top_dir") || canonical_top="$top_dir" + else + canonical_top="$top_dir" + fi + while IFS= read -r -d '' jar; do + name=$(basename "$jar") + for excluded in "${excluded_patterns[@]}"; do + # shellcheck disable=SC2254 # exclusion arguments are glob patterns + case "$name" in + $excluded) continue 2 ;; + esac + done + canonical_jar=$(canonical_path "$jar") || continue + [[ -f "$canonical_jar" ]] || continue + if [[ "$canonical_top" = "/" ]]; then + continue + fi + case "$canonical_jar" in + "$canonical_top" | "$canonical_top"/*) continue ;; + esac + printf '%s\n' "$canonical_jar" + done < <(find -P "$lib_dir" -name "$pattern" \ + ! -path "$top_dir/*" -print0) +} + function configure_riscv64_libatomic() { if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "riscv64" ]]; then return 0 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh new file mode 100755 index 0000000000..a1e3f51474 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +if [ "$#" -lt 3 ] || [ "$#" -gt 4 ]; then + echo "Usage: $0 [enforce]" >&2 + exit 2 +fi + +COMPONENT="$1" +PROVIDER="$2" +DATA_PATHS="$3" +ENFORCE="${4:-false}" +MARKER_NAME=".hugegraph-rocksdb-provider" + +fail() { + echo "Error: $*" >&2 + exit 1 +} + +case "$COMPONENT" in + server | pd | store) ;; + *) fail "invalid RocksDB component '$COMPONENT'" ;; +esac +case "$PROVIDER" in + rocksdb | topling) ;; + *) fail "invalid RocksDB provider '$PROVIDER'" ;; +esac +case "$ENFORCE" in + true | false) ;; + *) fail "provider marker enforcement must be true or false" ;; +esac + +EXPECTED_MARKER=$(printf '%s\n' \ + "format=1" \ + "component=$COMPONENT" \ + "provider=$PROVIDER") + +command -v flock >/dev/null 2>&1 || + fail "flock is required to verify RocksDB provider markers" + +verify_marker() { + local data_path="$1" + local pinned_path="$2" + local marker="$pinned_path/$MARKER_NAME" + local actual_marker + + if [ -L "$marker" ] || { [ -e "$marker" ] && [ ! -f "$marker" ]; }; then + fail "provider marker is not a regular file: $marker" + fi + if [ -f "$marker" ]; then + [ -r "$marker" ] || fail "provider marker is not readable: $marker" + actual_marker=$(<"$marker") + if [ "$actual_marker" != "$EXPECTED_MARKER" ]; then + fail "provider marker mismatch in $data_path; expected" \ + "$COMPONENT/$PROVIDER" + fi + return 0 + fi + + if [ "$PROVIDER" = "rocksdb" ] && [ "$ENFORCE" = "false" ]; then + echo "[provider-marker] legacy unmarked RocksDB path accepted: $data_path" + return 0 + fi + + if find -H "$pinned_path" -mindepth 1 -maxdepth 1 \ + ! -name lost+found -print -quit | grep -q .; then + fail "refusing unmarked non-empty data path: $data_path" + fi + + local temporary_marker + temporary_marker=$(mktemp "$pinned_path/.provider-marker.XXXXXX") + chmod 600 "$temporary_marker" + printf '%s\n' "$EXPECTED_MARKER" > "$temporary_marker" + # A bare launcher may claim a configured child while this deployment root + # was still empty. Once our temporary marker is visible it blocks new child + # claims; recheck after publishing it so an earlier child cannot be missed. + if find -H "$pinned_path" -mindepth 1 -maxdepth 1 ! -name lost+found \ + ! -name "$(basename "$temporary_marker")" ! -name "$MARKER_NAME" -print -quit | grep -q .; then + rm -f "$temporary_marker" + fail "data path changed while initializing provider marker: $data_path" + fi + if ! mv -n "$temporary_marker" "$marker"; then + rm -f "$temporary_marker" + fail "could not create provider marker: $marker" + fi + if [ -e "$temporary_marker" ]; then + rm -f "$temporary_marker" + fi + + actual_marker=$(<"$marker") + if [ "$actual_marker" != "$EXPECTED_MARKER" ]; then + fail "provider marker changed while initializing: $marker" + fi + echo "[provider-marker] initialized $COMPONENT/$PROVIDER at $data_path" +} + +IFS=',' read -r -a PATH_LIST <<<"$DATA_PATHS" +[ "${#PATH_LIST[@]}" -gt 0 ] || fail "no RocksDB data path configured" + +for raw_path in "${PATH_LIST[@]}"; do + data_path=$(printf '%s' "$raw_path" | + sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$data_path" ] || fail "empty RocksDB data path" + [[ "$data_path" == /* ]] || + fail "RocksDB data path must be absolute: $data_path" + [ "$data_path" != "/" ] || fail "RocksDB data path cannot be /" + + [ -d "$data_path" ] || + fail "RocksDB data path must be an existing directory: $data_path" + [ ! -L "$data_path" ] || + fail "RocksDB data path cannot be a symlink: $data_path" + + lexical_path=$(realpath -m -s "$data_path") + physical_path=$(realpath -m "$data_path") + [ "$lexical_path" = "$physical_path" ] || + fail "RocksDB data path contains a symlink: $data_path" + + exec {data_path_fd}<"$data_path" + flock -x "$data_path_fd" || + fail "could not lock RocksDB data path: $data_path" + pinned_path="/proc/self/fd/$data_path_fd" + [ -d "$pinned_path" ] || + fail "could not pin RocksDB data path: $data_path" + + configured_identity=$(stat -Lc '%d:%i' "$data_path") + pinned_identity=$(stat -Lc '%d:%i' "$pinned_path") + [ "$configured_identity" = "$pinned_identity" ] || + fail "RocksDB data path changed while locking: $data_path" + + verify_marker "$data_path" "$pinned_path" + + configured_identity=$(stat -Lc '%d:%i' "$data_path") + pinned_identity=$(stat -Lc '%d:%i' "$pinned_path") + [ "$configured_identity" = "$pinned_identity" ] || + fail "RocksDB data path changed while verifying: $data_path" + + flock -u "$data_path_fd" + exec {data_path_fd}<&- +done diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties index 26adfe0183..f424cdb1f8 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/graphs/hugegraph.properties @@ -43,6 +43,10 @@ search.text_analyzer_mode=INDEX # rocksdb backend config #rocksdb.data_path=/path/to/disk #rocksdb.wal_path=/path/to/disk +# To enable ToplingDB: uncomment the line below (requires ToplingDB JAR in lib/topling/) +# The startup script detects this setting and handles JAR swap + env var setup automatically. +# ToplingDB config: conf/toplingdb.yaml (auto-loaded via TOPLINGDB_EASY_MIGRATE_CONF env var) +#rocksdb.provider=topling # hbase backend config #hbase.hosts=localhost diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml new file mode 100644 index 0000000000..f7c902cf4e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library/rocksdb_resource + listening_ports: '127.0.0.1:2011' + auto_start_http: false +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With + # memtable_as_log_index enabled, every batch write fails: + # "WriteBatch has no mmap wal". Keep the option false for this client. + # convert_to_sst is still Topling-specific. Do not open data written by + # this profile with the standard RocksDB provider; restore a snapshot + # taken before migration instead. + compaction_readahead_size: 0 + memtable_as_log_index: false diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar b/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar new file mode 100644 index 0000000000..61c401bb01 Binary files /dev/null and b/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar differ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/RocksDBRuntimeSmokeTest.java b/hugegraph-server/hugegraph-dist/src/assembly/travis/RocksDBRuntimeSmokeTest.java new file mode 100644 index 0000000000..422c7091ad --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/RocksDBRuntimeSmokeTest.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksIterator; + +public final class RocksDBRuntimeSmokeTest { + + private static final byte[] CF = bytes("runtime-smoke"); + private static final byte[] KEY = bytes("key"); + private static final byte[] VALUE = bytes("value-before-restart"); + private static final byte[] RECREATED_VALUE = bytes("value-after-recreate"); + + public static void main(String[] args) throws Exception { + if (args.length != 4) { + throw new IllegalArgumentException( + "Usage: "); + } + + String provider = args[0]; + String dbPath = args[1]; + String expectedNativePath = args[2]; + String mode = args[3]; + if (!"probe".equals(mode) && !"lifecycle".equals(mode)) { + throw new IllegalArgumentException("Unsupported mode: " + mode); + } + phase("runtime-check"); + verifyProvider(provider); + verifyEasyMigrateConfig(provider); + RocksDB.loadLibrary(); + verifyNativeLibrary(expectedNativePath); + + if ("probe".equals(mode)) { + probeReadWrite(dbPath); + } else { + createAndWrite(dbPath); + reopenDropAndRecreate(dbPath); + } + phase("complete"); + System.out.printf("Runtime smoke test passed: provider=%s, version=%s%n", + provider, RocksDB.rocksdbVersion()); + } + + private static void phase(String phase) { + System.out.println("Topling diagnostic phase: " + phase); + System.out.flush(); + } + + private static void probeReadWrite(String dbPath) throws Exception { + try (Options options = new Options().setCreateIfMissing(true); + RocksDB db = RocksDB.open(options, dbPath)) { + db.put(KEY, VALUE); + assertBytes(VALUE, db.get(KEY), "probe read"); + try (RocksIterator iterator = db.newIterator()) { + iterator.seekToFirst(); + if (!iterator.isValid()) { + throw new AssertionError("probe iterator returned no data"); + } + assertBytes(KEY, iterator.key(), "probe iterator key"); + assertBytes(VALUE, iterator.value(), "probe iterator value"); + } + } + } + + private static void verifyProvider(String provider) { + boolean hasToplingApi; + try { + Class.forName("org.rocksdb.SidePluginRepo", false, + RocksDBRuntimeSmokeTest.class.getClassLoader()); + hasToplingApi = true; + } catch (ClassNotFoundException ignored) { + hasToplingApi = false; + } + + if (!"rocksdb".equals(provider) && !"topling".equals(provider)) { + throw new IllegalArgumentException("Unsupported provider: " + provider); + } + if ("rocksdb".equals(provider) && hasToplingApi) { + throw new IllegalStateException("Standard provider loaded a Topling JAR"); + } + if ("topling".equals(provider) && !hasToplingApi) { + throw new IllegalStateException("Topling provider loaded no Topling API"); + } + } + + private static void verifyEasyMigrateConfig(String provider) { + String config = System.getenv("TOPLINGDB_EASY_MIGRATE_CONF"); + if ("rocksdb".equals(provider)) { + if (config != null && !config.isEmpty()) { + throw new IllegalStateException( + "Standard RocksDB test must not enable Easy Migrate"); + } + return; + } + if (config == null || config.isEmpty()) { + throw new IllegalStateException( + "ToplingDB functional test requires Easy Migrate config"); + } + if (!Files.isReadable(Paths.get(config))) { + throw new IllegalStateException( + "Easy Migrate config is not readable: " + config); + } + System.out.println("Verified Easy Migrate config: " + + Paths.get(config).toAbsolutePath().normalize()); + } + + private static void verifyNativeLibrary(String expectedNativePath) + throws Exception { + if ("none".equals(expectedNativePath)) { + return; + } + String maps = new String(Files.readAllBytes(Paths.get("/proc/self/maps")), + StandardCharsets.UTF_8); + String absolutePath = Paths.get(expectedNativePath).toAbsolutePath() + .normalize().toString(); + if (!maps.contains(absolutePath)) { + throw new IllegalStateException("Expected native library is not mapped: " + + absolutePath); + } + for (String line : maps.split("\\R")) { + if (line.contains("librocksdbjni") && !line.contains(absolutePath)) { + throw new IllegalStateException( + "An unexpected RocksDB JNI library is also mapped: " + line); + } + } + System.out.println("Verified native library: " + absolutePath); + } + + private static void createAndWrite(String dbPath) throws Exception { + try (DiagnosticResources resources = new DiagnosticResources()) { + try { + Options options = resources.add(new Options().setCreateIfMissing(true)); + RocksDB db = resources.add(RocksDB.open(options, dbPath)); + ColumnFamilyOptions cfOptions = resources.add(new ColumnFamilyOptions()); + ColumnFamilyHandle handle = resources.add(db.createColumnFamily( + new ColumnFamilyDescriptor(CF, cfOptions))); + db.put(handle, KEY, VALUE); + assertBytes(VALUE, db.get(handle, KEY), "initial read"); + try (RocksIterator iterator = db.newIterator(handle)) { + iterator.seekToFirst(); + if (!iterator.isValid()) { + throw new AssertionError("iterator returned no data"); + } + assertBytes(KEY, iterator.key(), "iterator key"); + assertBytes(VALUE, iterator.value(), "iterator value"); + } + phase("cf-lifecycle"); + } catch (Exception | Error failure) { + // Record before native cleanup can abort and hide the Java error. + failed(failure); + throw failure; + } + } + } + + private static void reopenDropAndRecreate(String dbPath) throws Exception { + phase("runtime-check"); + List descriptors = new ArrayList<>(); + try (Options options = new Options()) { + for (byte[] name : RocksDB.listColumnFamilies(options, dbPath)) { + descriptors.add(new ColumnFamilyDescriptor(name)); + } + } + + try (DiagnosticResources resources = new DiagnosticResources()) { + try { + DBOptions options = resources.add(new DBOptions().setCreateIfMissing(false)); + List handles = new ArrayList<>(); + RocksDB db = resources.add(RocksDB.open(options, dbPath, descriptors, handles)); + for (ColumnFamilyHandle handle : handles) { + resources.add(handle); + } + ColumnFamilyHandle smoke = findHandle(descriptors, handles, CF); + assertBytes(VALUE, db.get(smoke, KEY), "read after reopen"); + phase("cf-lifecycle"); + db.dropColumnFamily(smoke); + smoke.close(); + resources.remove(smoke); + + ColumnFamilyOptions cfOptions = resources.add(new ColumnFamilyOptions()); + ColumnFamilyHandle recreated = resources.add(db.createColumnFamily( + new ColumnFamilyDescriptor(CF, cfOptions))); + phase("runtime-check"); + db.put(recreated, KEY, RECREATED_VALUE); + assertBytes(RECREATED_VALUE, db.get(recreated, KEY), + "read after CF recreation"); + phase("cf-lifecycle"); + } catch (Exception | Error failure) { + failed(failure); + throw failure; + } + } + } + + private static void failed(Throwable failure) { + phase("failed"); + failure.printStackTrace(System.err); + System.err.flush(); + } + + // Mark a thrown close failure before closing the next resource: otherwise a + // secondary native abort could hide it and look like the known CF assertion. + static final class DiagnosticResources implements AutoCloseable { + + private final List resources = new ArrayList<>(); + + T add(T resource) { + this.resources.add(resource); + return resource; + } + + void remove(AutoCloseable resource) { + // Native-backed equals() may dereference a handle after close(). + for (int i = 0; i < this.resources.size(); i++) { + if (this.resources.get(i) == resource) { + this.resources.remove(i); + return; + } + } + } + + @Override + public void close() throws Exception { + Throwable failure = null; + for (int i = this.resources.size() - 1; i >= 0; i--) { + try { + this.resources.get(i).close(); + } catch (Exception | Error current) { + failed(current); + if (failure == null) { + failure = current; + } else { + failure.addSuppressed(current); + } + } + } + if (failure instanceof Exception) { + throw (Exception) failure; + } + if (failure != null) { + throw (Error) failure; + } + } + } + + private static ColumnFamilyHandle findHandle( + List descriptors, + List handles, + byte[] name) { + for (int i = 0; i < descriptors.size(); i++) { + if (Arrays.equals(name, descriptors.get(i).getName())) { + return handles.get(i); + } + } + throw new IllegalStateException("Column family was not reopened"); + } + + private static void assertBytes(byte[] expected, byte[] actual, + String operation) { + if (!Arrays.equals(expected, actual)) { + throw new AssertionError(operation + " returned unexpected data"); + } + } + + private static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private RocksDBRuntimeSmokeTest() { + } +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/ToplingDiagnosticResourcesTest.java b/hugegraph-server/hugegraph-dist/src/assembly/travis/ToplingDiagnosticResourcesTest.java new file mode 100644 index 0000000000..76216eb69e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/ToplingDiagnosticResourcesTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; + +public final class ToplingDiagnosticResourcesTest { + + public static void main(String[] args) throws Exception { + testIdentityRemoval(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream originalOut = System.out; + PrintStream originalErr = System.err; + IllegalStateException expected = new IllegalStateException("injected close failure"); + boolean[] nextCleanupRan = {false}; + try (PrintStream capture = new PrintStream(output, true, StandardCharsets.UTF_8.name())) { + System.setOut(capture); + System.setErr(capture); + try (RocksDBRuntimeSmokeTest.DiagnosticResources resources = + new RocksDBRuntimeSmokeTest.DiagnosticResources()) { + resources.add(() -> { + String log = output.toString(StandardCharsets.UTF_8.name()); + if (!log.contains("Topling diagnostic phase: failed") || + !log.contains("injected close failure")) { + throw new AssertionError("failure was hidden before subsequent cleanup"); + } + nextCleanupRan[0] = true; + }); + resources.add(() -> { + throw expected; + }); + } catch (IllegalStateException failure) { + if (failure != expected || failure.getSuppressed().length != 0) { + throw new AssertionError("unexpected cleanup failure", failure); + } + } + if (!nextCleanupRan[0]) { + throw new AssertionError("remaining resource was not closed"); + } + } finally { + System.setOut(originalOut); + System.setErr(originalErr); + } + System.out.println("PASS: Java close failure recorded before remaining cleanup"); + } + + private static void testIdentityRemoval() throws Exception { + IdentityOnlyResource remaining = new IdentityOnlyResource(); + IdentityOnlyResource removed = new IdentityOnlyResource(); + try (RocksDBRuntimeSmokeTest.DiagnosticResources resources = + new RocksDBRuntimeSmokeTest.DiagnosticResources()) { + resources.add(remaining); + resources.add(removed); + removed.close(); + resources.remove(removed); + } + if (!remaining.closed) { + throw new AssertionError("remaining resource was not closed"); + } + System.out.println("PASS: closed resources removed by identity without equals"); + } + + private static final class IdentityOnlyResource implements AutoCloseable { + + private boolean closed; + + @Override + public boolean equals(Object other) { + throw new AssertionError("native-backed equals must not be called"); + } + + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + @Override + public void close() { + if (this.closed) { + throw new AssertionError("resource was closed twice"); + } + this.closed = true; + } + } + + private ToplingDiagnosticResourcesTest() { + } +} diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh new file mode 100755 index 0000000000..c5719254a4 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-deps.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -ev + +if [ -f /etc/debian_version ]; then + sudo apt-get update && sudo apt-get install -y liburing-dev libaio-dev libjemalloc-dev +elif [ -f /etc/redhat-release ]; then + sudo yum install -y liburing-devel libaio-devel jemalloc-devel +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh new file mode 100755 index 0000000000..384a7ab274 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/install-rocksdb.sh @@ -0,0 +1,197 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ "$(uname -s)" != "Linux" ]; then + echo "[install-rocksdb] Skip native preload on non-Linux platform: $(uname -s)" + return 0 2>/dev/null || exit 0 +fi + +ORIG_SHELL_FLAGS="$-" +ORIG_PIPEFAIL="$(set -o | awk '$1 == "pipefail" { print $2 }')" +ORIG_ERR_TRAP="$(trap -p ERR)" +ORIG_EXIT_TRAP="$(trap -p EXIT)" +# Save original IFS to avoid leaking into parent shell when sourced +ORIG_IFS="${IFS}" +set -Eeuo pipefail +IFS=$'\n\t' + +install_rocksdb_restore_state() { + local exit_status=$? + + # Prevent recursive execution while restoring the caller's EXIT trap. + trap - EXIT + IFS="$ORIG_IFS" + if [ -n "$ORIG_ERR_TRAP" ]; then + eval "$ORIG_ERR_TRAP" + else + trap - ERR + fi + if [ -n "$ORIG_EXIT_TRAP" ]; then + eval "$ORIG_EXIT_TRAP" + else + trap - EXIT + fi + case "$ORIG_SHELL_FLAGS" in *e*) set -e ;; *) set +e ;; esac + case "$ORIG_SHELL_FLAGS" in *u*) set -u ;; *) set +u ;; esac + case "$ORIG_SHELL_FLAGS" in *E*) set -E ;; *) set +E ;; esac + if [ "$ORIG_PIPEFAIL" = "on" ]; then + set -o pipefail + else + set +o pipefail + fi + return "$exit_status" +} +trap install_rocksdb_restore_state EXIT + +# Unified error capture for easy positioning +trap 'echo "[install-rocksdb] error at line ${LINENO}: ${BASH_COMMAND}" >&2' ERR + +VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) +COMPONENT="${1:-server}" +SERVER_VERSION_DIR="$(pwd)/hugegraph-server/apache-hugegraph-server-$VERSION" +TOPLING_SOURCE_LIB="$(pwd)/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling" + +case "$COMPONENT" in + server | hstore) + COMPONENT_VERSION_DIR="$SERVER_VERSION_DIR" + ;; + pd) + COMPONENT_VERSION_DIR="$(pwd)/hugegraph-pd/apache-hugegraph-pd-$VERSION" + ;; + store) + COMPONENT_VERSION_DIR="$(pwd)/hugegraph-store/apache-hugegraph-store-$VERSION" + ;; + *) + echo "Error: unsupported component '$COMPONENT' (expected server, pd, store, or hstore)" >&2 + exit 1 + ;; +esac +COMPONENT_BIN="$COMPONENT_VERSION_DIR/bin" +COMPONENT_LIB="$COMPONENT_VERSION_DIR/lib" + +if [ ! -d "$COMPONENT_VERSION_DIR" ]; then + echo "Error: component dir not found: $COMPONENT_VERSION_DIR" >&2 + exit 1 +fi + +detect_rocksdb_provider() { + local conf_dir="$1" + local file + local -a values=() + local -a unique_values=() + local value existing duplicate conflicts + + for file in "$conf_dir"/graphs/*.properties; do + [ -f "$file" ] || continue + while IFS= read -r value; do + values[${#values[@]}]="$value" + done < <( + awk ' + /^[[:space:]]*#/ { next } + /^[[:space:]]*rocksdb\.provider[[:space:]]*=/ { + value = $0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + print value + } + ' "$file" + ) + done + for file in "$conf_dir"/application*.yml; do + [ -f "$file" ] || continue + while IFS= read -r value; do + values[${#values[@]}]="$value" + done < <( + awk ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + /^[^[:space:]#][^:]*:/ { + in_rocksdb = ($0 ~ /^rocksdb[[:space:]]*:/) + } + in_rocksdb && /^[[:space:]]+provider[[:space:]]*:/ { + value = $0 + sub(/^[^:]*:[[:space:]]*/, "", value) + sub(/[[:space:]]+#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (value ~ /^"[^"]*"$/ || value ~ /^'\''[^'\'']*'\''$/) { + value = substr(value, 2, length(value) - 2) + } + print value + } + ' "$file" + ) + done + + for value in "${values[@]}"; do + duplicate=false + for existing in "${unique_values[@]}"; do + if [ "$existing" = "$value" ]; then + duplicate=true + break + fi + done + if [ "$duplicate" = false ]; then + unique_values+=("$value") + fi + done + + if [ "${#unique_values[@]}" -gt 1 ]; then + conflicts=$(IFS=,; echo "${unique_values[*]}") + echo "Error: conflicting rocksdb.provider values: $conflicts" >&2 + return 1 + fi + local provider="${unique_values[0]:-rocksdb}" + case "$provider" in + rocksdb | topling) + echo "$provider" + ;; + *) + echo "Error: invalid rocksdb.provider '$provider'; expected rocksdb or topling" >&2 + return 1 + ;; + esac +} + +PROVIDER=$(detect_rocksdb_provider "$COMPONENT_VERSION_DIR/conf") || exit 1 + +if [ "$PROVIDER" = "topling" ]; then + if [ ! -x "$COMPONENT_BIN/prepare-topling.sh" ]; then + echo "Error: prepare-topling.sh not found under: $COMPONENT_BIN" >&2 + exit 1 + fi + + TOPLING_JARS=("$TOPLING_SOURCE_LIB"/rocksdbjni*.jar) + if [ "${#TOPLING_JARS[@]}" -ne 1 ] || [ ! -f "${TOPLING_JARS[0]}" ]; then + echo "Error: expected exactly one ToplingDB JAR under: $TOPLING_SOURCE_LIB" >&2 + exit 1 + fi + mkdir -p "$COMPONENT_LIB/topling" + rm -f "$COMPONENT_LIB"/topling/rocksdbjni*.jar + cp "${TOPLING_JARS[0]}" "$COMPONENT_LIB/topling/" + "$COMPONENT_BIN/prepare-topling.sh" +else + echo "[install-rocksdb] $COMPONENT uses rocksdb provider (or unset)," \ + "skipping native preload" +fi + +unset -f detect_rocksdb_provider + +# A sourced script does not trigger EXIT on normal return, so restore explicitly. +# The EXIT trap above covers exit and errexit failure paths. +install_rocksdb_restore_state +unset -f install_rocksdb_restore_state diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh index 7ffa6b8685..089477cae9 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -67,6 +67,9 @@ JACOCO_PORT=36320 mvn package -Dmaven.test.skip=true -ntp +# install rocksdb +source "$TRAVIS_DIR/install-rocksdb.sh" server + if [[ ! -e "$SERVER_DIR/lib/ikanalyzer-2012_u6.jar" ]]; then download_to_dir "$SERVER_DIR/lib/" \ "https://raw.githubusercontent.com/apache/hugegraph-doc/ik_binary/dist/server/ikanalyzer-2012_u6.jar" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh index a95d2f0806..b1f6dbdf36 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-core-test.sh @@ -18,5 +18,9 @@ set -ev BACKEND=$1 +TRAVIS_DIR=$(dirname $0) + +# install rocksdb +source "$TRAVIS_DIR/install-rocksdb.sh" server mvn test -pl hugegraph-server/hugegraph-test -am -P core-test,$BACKEND diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-topling-native-diagnostic.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-topling-native-diagnostic.sh new file mode 100755 index 0000000000..d0b2782a34 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-topling-native-diagnostic.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +LOG_DIR="${1:?Usage: $0 [args...]}" +shift +if [ "$#" -eq 0 ]; then + echo 'Error: runtime command is required' >&2 + exit 1 +fi +mkdir -p "$LOG_DIR" + +report() { + local result="$1" status="$2" exit_status="$3" + { + echo '### Topling native lifecycle diagnostic' + echo + echo "Result: $result" + echo "Runtime exit status: $status" + if [ "$result" = known-cf-assertion ]; then + echo 'Verified runtime prerequisites passed; the synthetic CF phase hit' + echo 'the exact known assertion tracked by hugegraph/hugegraph#212.' + echo 'This diagnostic does not replace Linux service lifecycle acceptance.' + fi + echo 'Full probe/lifecycle logs and selected JNI SHA-256 identities are retained in the diagnostic artifact.' + } > "$LOG_DIR/report.md" + cat "$LOG_DIR/report.md" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$LOG_DIR/report.md" >> "$GITHUB_STEP_SUMMARY" + fi + exit "$exit_status" +} + +failure_pattern='verify\(.*\) failed:|assertion.*failed|db not closed|Exception|Error:|Error([[:space:]]|\]|$)|Fatal' +# This producer warning means the faster DirectByteBuffer constructor is absent. +# It is not a thrown probe failure; successful read/iteration/close is still required. +optional_warning='^WARN: access java[.]nio[.]DirectByteBuffer failed: java[.]lang[.]NoSuchMethodException: no such constructor: java[.]nio[.]DirectByteBuffer[.]\(long,long\)void/newInvokeSpecial$' +diagnostic_errors() { + grep -Ev "$optional_warning" "$1" | grep -Ei "$failure_pattern" || true +} + +# Run the mandatory probe in a separate process. Even a known assertion here +# fails: only the subsequent synthetic CF sequence has a narrow waiver. +set +e +"$@" probe 2>&1 | tee "$LOG_DIR/probe.log" +probe_status=("${PIPESTATUS[@]}") +set -e +if [ "${probe_status[1]}" -ne 0 ]; then + report log-capture-failed "${probe_status[1]}" 1 +fi +probe_errors=$(diagnostic_errors "$LOG_DIR/probe.log") +probe_phase=$(sed -n 's/^Topling diagnostic phase: //p' "$LOG_DIR/probe.log" | tail -1) +if [ "${probe_status[0]}" -ne 0 ] || [ "$probe_phase" != complete ] || + [ -n "$probe_errors" ]; then + report prerequisite-failed "${probe_status[0]}" 1 +fi + +set +e +"$@" lifecycle 2>&1 | tee "$LOG_DIR/lifecycle.log" +lifecycle_status=("${PIPESTATUS[@]}") +set -e +status="${lifecycle_status[0]}" +if [ "${lifecycle_status[1]}" -ne 0 ]; then + report log-capture-failed "${lifecycle_status[1]}" 1 +fi +phase=$(sed -n 's/^Topling diagnostic phase: //p' "$LOG_DIR/lifecycle.log" | tail -1) +known_pattern=': verify\(.*\) failed: cfh must in cfh_to_view !$' +# Do not waive a known abort accompanied by a different failure (including +# a Java failure which triggers a second assertion while unwinding resources). +unknown_errors=$(diagnostic_errors "$LOG_DIR/lifecycle.log" | + grep -Ev "$known_pattern" || true) +if [ "$status" -eq 0 ] && [ "$phase" = complete ] && [ -z "$unknown_errors" ] && + ! grep -Eq "$known_pattern" "$LOG_DIR/lifecycle.log"; then + report passed "$status" 0 +fi +if [ "$status" -eq 134 ] && [ "$phase" = cf-lifecycle ] && [ -z "$unknown_errors" ] && + grep -Eq "$known_pattern" "$LOG_DIR/lifecycle.log"; then + report known-cf-assertion "$status" 0 +fi +report unknown-failure "$status" 1 diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh index 5fe9b476b3..1f35812522 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-unit-test.sh @@ -19,6 +19,9 @@ set -ev BACKEND=$1 +# install rocksdb +source "$TRAVIS_DIR/install-rocksdb.sh" server + if [[ "$BACKEND" == "memory" ]]; then mvn test -pl hugegraph-server/hugegraph-test -am -P unit-test fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 796d69c83b..452042b915 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -74,6 +74,11 @@ fi TEMP_DIR=$(mktemp -d) SECURITY_PROPERTIES_BACKUP="${TEMP_DIR}/java-security.properties" +# The provider selector reads REST configuration before invoking the JVM. +# Keep it valid while a missing Gremlin file stops the real server before bind. +FIXTURE_REST_PROPERTIES="${TEMP_DIR}/rest-server.properties" +mkdir "${TEMP_DIR}/graphs" +printf 'graphs=%s\n' "${TEMP_DIR}/graphs" > "$FIXTURE_REST_PROPERTIES" cleanup() { if [[ -d "$SECURITY_PROPERTIES" ]]; then @@ -127,11 +132,12 @@ assert_invalid_security_properties() { assert_reached_server_startup() { local error_file="$1" local message="$2" - grep -Fq "Failed to load yaml config file" "$error_file" || fail "$message" - grep -Fq "org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap.main" \ - "$error_file" || fail "$message" - grep -Fq "org.apache.hugegraph.dist.HugeGraphServer.main" \ - "$error_file" || fail "$message" + if ! grep -Fq "Failed to load yaml config file" "$error_file" || + ! grep -Fq "org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap.main" "$error_file" || + ! grep -Fq "org.apache.hugegraph.dist.HugeGraphServer.main" "$error_file"; then + cat "$error_file" >&2 + fail "$message" + fi } assert_clean_bootstrap_error() { @@ -229,7 +235,7 @@ assert_launcher_accepts_security_properties() { local error_file="${TEMP_DIR}/launcher-valid.err" if JAVA_OPTIONS="" STDOUT_MODE=true "$SERVER_SCRIPT" \ "${TEMP_DIR}/missing-gremlin.yaml" \ - "${TEMP_DIR}/missing-rest.properties" true \ + "$FIXTURE_REST_PROPERTIES" true \ "-Djava.security.properties=${properties_path}" \ >/dev/null 2>"$error_file"; then fail "server unexpectedly started with missing configuration" @@ -248,7 +254,7 @@ assert_launcher_skips_security_validation() { -Djava.security.properties=${INFINITE_PROPERTIES}" \ JAVA_OPTIONS="" STDOUT_MODE=true "$SERVER_SCRIPT" \ "${TEMP_DIR}/missing-gremlin.yaml" \ - "${TEMP_DIR}/missing-rest.properties" false \ + "$FIXTURE_REST_PROPERTIES" false \ >/dev/null 2>"$error_file"; then fail "server unexpectedly started with missing configuration" fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh index ed39b67492..5d28c711a8 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-pd-docker-entrypoint.sh @@ -35,8 +35,11 @@ FAIL=0 [[ -f "${ENTRYPOINT}" ]] || { echo "entrypoint not found at ${ENTRYPOINT}" >&2; exit 1; } command -v python3 >/dev/null || { echo "python3 required" >&2; exit 1; } -mkdir -p "${TMP_DIR}/bin" +mkdir -p "${TMP_DIR}/bin" "${TMP_DIR}/pd_data" cp "${ENTRYPOINT}" "${TMP_DIR}/docker-entrypoint.sh" +cp "$(dirname "${BASH_SOURCE[0]}")/../static/bin/verify-rocksdb-provider.sh" \ + "${TMP_DIR}/bin/verify-rocksdb-provider.sh" +chmod +x "${TMP_DIR}/bin/verify-rocksdb-provider.sh" # Stand in for the launcher: record the generated config instead of starting PD cat > "${TMP_DIR}/bin/start-hugegraph-pd.sh" <<'STUB' #!/usr/bin/env bash diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-rocksdb-runtime.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-rocksdb-runtime.sh new file mode 100755 index 0000000000..eaecdcd8ac --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-rocksdb-runtime.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +PROVIDER="${1:?Usage: $0 [probe|lifecycle]}" +COMPONENT_DIR="${2:?Usage: $0 [probe|lifecycle]}" +MODE="${3:-lifecycle}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ "$PROVIDER" = topling ] && [ -f "$COMPONENT_DIR/bin/preload-topling.sh" ]; then + # shellcheck source=/dev/null + source "$COMPONENT_DIR/bin/preload-topling.sh" +fi + +TEST_ROOT=$(mktemp -d /tmp/hugegraph-rocksdb-runtime.XXXXXX) +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +case "$PROVIDER" in + rocksdb) + EXPECTED_NATIVE_PATH=none + ;; + topling) + EXPECTED_NATIVE_PATH="$COMPONENT_DIR/library/librocksdbjni-linux64.so" + if [ ! -f "$EXPECTED_NATIVE_PATH" ]; then + echo "Error: Topling native library was not installed for the component" >&2 + exit 1 + fi + ;; + *) + echo "Error: unsupported provider '$PROVIDER'" >&2 + exit 1 + ;; +esac + +if compgen -G "$COMPONENT_DIR/lib/rocksdbjni*.jar" >/dev/null; then + if [ "$PROVIDER" = "topling" ]; then + JAR="${TOPLING_RUNTIME_CLASSPATH:-}" + else + JAR=$(ls -1 "$COMPONENT_DIR"/lib/rocksdbjni*.jar 2>/dev/null | + sort -V | tail -1 || true) + fi +else + if [ "$PROVIDER" = "topling" ]; then + JAR=$(ls -1 "$COMPONENT_DIR"/lib/topling/rocksdbjni*.jar 2>/dev/null | + sort -V | tail -1 || true) + else + BOOT_JAR=$(ls -1 "$COMPONENT_DIR"/lib/*.jar 2>/dev/null | + sort -V | tail -1 || true) + NESTED_JAR=$(unzip -Z1 "$BOOT_JAR" 'BOOT-INF/lib/rocksdbjni*.jar' | + sort -V | tail -1 || true) + if [ -z "$NESTED_JAR" ]; then + echo "Error: no embedded rocksdbjni JAR found in $BOOT_JAR" >&2 + exit 1 + fi + JAR="$TEST_ROOT/rocksdbjni.jar" + unzip -p "$BOOT_JAR" "$NESTED_JAR" > "$JAR" + fi +fi +if [ -z "$JAR" ]; then + echo "Error: no rocksdbjni JAR found for provider '$PROVIDER'" >&2 + exit 1 +fi + +if [ "$PROVIDER" = "topling" ]; then + EASY_MIGRATE_CONF="${TOPLINGDB_EASY_MIGRATE_CONF:-}" + if [ -z "$EASY_MIGRATE_CONF" ]; then + for candidate in \ + "$COMPONENT_DIR/conf/toplingdb.yaml" \ + "$COMPONENT_DIR/conf/rocksdb_store.yaml" \ + "$COMPONENT_DIR/conf/rocksdb_pd.yaml"; do + if [ -f "$candidate" ]; then + EASY_MIGRATE_CONF="$candidate" + break + fi + done + fi + if [ -z "$EASY_MIGRATE_CONF" ] || [ ! -r "$EASY_MIGRATE_CONF" ]; then + echo "Error: readable ToplingDB Easy Migrate config is required" >&2 + exit 1 + fi + + echo "Selected JNI JAR: $JAR" + echo "Selected JNI native library: $EXPECTED_NATIVE_PATH" + sha256sum "$JAR" "$EXPECTED_NATIVE_PATH" + + NATIVE_DIR="$(dirname "$EXPECTED_NATIVE_PATH")" + TEST_LD_LIBRARY_PATH="$NATIVE_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + TEST_LD_PRELOAD="${LD_PRELOAD:-}" + case ":$TEST_LD_PRELOAD:" in + *":$EXPECTED_NATIVE_PATH:"*) ;; + *) TEST_LD_PRELOAD="$EXPECTED_NATIVE_PATH${TEST_LD_PRELOAD:+:$TEST_LD_PRELOAD}" ;; + esac + + TOPLINGDB_EASY_MIGRATE_CONF="$EASY_MIGRATE_CONF" \ + LD_LIBRARY_PATH="$TEST_LD_LIBRARY_PATH" \ + LD_PRELOAD="$TEST_LD_PRELOAD" \ + java -cp "$JAR" "$SCRIPT_DIR/RocksDBRuntimeSmokeTest.java" \ + "$PROVIDER" "$TEST_ROOT/db" "$EXPECTED_NATIVE_PATH" "$MODE" +else + env -u TOPLINGDB_EASY_MIGRATE_CONF \ + java -cp "$JAR" "$SCRIPT_DIR/RocksDBRuntimeSmokeTest.java" \ + "$PROVIDER" "$TEST_ROOT/db" "$EXPECTED_NATIVE_PATH" "$MODE" +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh index 9f0bcfaa63..25c502c0e8 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh @@ -26,7 +26,7 @@ # # Note: Tests require crontab access. On macOS Catalina+ this may # require Full Disk Access permission in System Preferences. -# Note: Assumes rocksdb backend. init-store.sh is run automatically if needed. +# Note: Assumes rocksdb backend. init-store.sh checks initialization on every run. set -uo pipefail @@ -42,6 +42,8 @@ WAIT_TIMEOUT=30 # seconds for timeout on wait calls PASS=0 FAIL=0 ERRORS=() +STARTUP_LOGS_REPORTED=false +WAIT_TIMED_OUT=false GREEN='\033[0;32m' RED='\033[0;31m' @@ -57,6 +59,17 @@ fail() { echo -e "${RED} FAIL${NC} $1" ERRORS+=("$1") FAIL=$((FAIL + 1)) + if [[ "$STARTUP_LOGS_REPORTED" == false ]]; then + local log + for log in "$HUGEGRAPH_ROOT/logs/hugegraph-server.log" \ + "$HUGEGRAPH_ROOT/logs/hugegraph-server-stdout.log"; do + if [[ -f "$log" ]]; then + echo "--- $log (last 80 lines) ---" + tail -n 80 "$log" + fi + done + STARTUP_LOGS_REPORTED=true + fi } info() { @@ -66,6 +79,12 @@ info() { section() { echo "" echo "── $1 ──" + STARTUP_LOGS_REPORTED=false +} + +server_process_running() { + [[ "$1" =~ ^[0-9]+$ ]] && + ps -p "$1" -o args= -ww 2>/dev/null | grep -Fq -- '-Dname=HugeGraphServer' } # Kill server, clear pid file, kill ports, clear crontab monitor entry @@ -133,14 +152,25 @@ wait_for_pid_file() { # bash's built-in wait can track it as a child process wait_script_exit() { local script_pid="$1" + local timeout_marker + WAIT_TIMED_OUT=false + timeout_marker=$(mktemp) || { WAIT_TIMED_OUT=true; return 1; } # background killer fires after timeout if process hasn't exited yet - ( sleep "$WAIT_TIMEOUT" && kill "$script_pid" 2>/dev/null ) & + ( + sleep "$WAIT_TIMEOUT" + if kill -0 "$script_pid" 2>/dev/null; then + printf 'timeout\n' > "$timeout_marker" + kill "$script_pid" 2>/dev/null + fi + ) & local killer_pid=$! wait "$script_pid" 2>/dev/null local exit_code=$? # cancel the killer if process already exited kill "$killer_pid" 2>/dev/null || true wait "$killer_pid" 2>/dev/null || true + [[ -s "$timeout_marker" ]] && WAIT_TIMED_OUT=true + rm -f "$timeout_marker" return $exit_code } @@ -189,21 +219,22 @@ if [[ -z "${JAVA_HOME:-}" ]]; then echo " Export JAVA_HOME before running this script" fi -# Run init-store.sh once if RocksDB has not been initialized yet -if [[ ! -d "$HUGEGRAPH_ROOT/rocksdb-data" ]]; then - info "RocksDB not initialized — running init-store.sh..." - if "$BIN/init-store.sh" >/dev/null 2>&1; then - pass "init-store.sh completed successfully" - else - echo -e "${RED}ERROR:${NC} init-store.sh failed. Cannot run tests." - exit 1 - fi +cleanup + +# Provider admission can create data roots before the backend has any tables. +# Let init-store inspect the actual backend; it already handles initialized stores. +info "Checking backend initialization with init-store.sh..." +INIT_LOG=$(mktemp) +if "$BIN/init-store.sh" >"$INIT_LOG" 2>&1; then + pass "init-store.sh completed successfully" + rm -f "$INIT_LOG" else - info "RocksDB already initialized, skipping init-store.sh" + echo -e "${RED}ERROR:${NC} init-store.sh failed. Cannot run tests." + cat "$INIT_LOG" + rm -f "$INIT_LOG" + exit 1 fi -cleanup - # ── test 1: daemon mode regression ─────────────────────────────────────────── section "Test 1 — daemon mode regression" @@ -227,7 +258,7 @@ else fail "bin/pid is empty or missing after daemon start" fi -if [[ -n "$DAEMON_PID" ]] && ps -p "$DAEMON_PID" >/dev/null 2>&1; then +if server_process_running "$DAEMON_PID"; then pass "Java process is running (pid $DAEMON_PID)" else fail "Java process not found for pid '$DAEMON_PID'" @@ -263,7 +294,7 @@ info "Waiting up to ${STARTUP_WAIT}s for server to come up..." if wait_for_server; then info "Server is up" else - info "Server did not respond — continuing to check blocking behavior" + fail "server did not become ready before the blocking test" fi # Primary assertion: script must still be running while Java is alive @@ -283,7 +314,7 @@ else info "pid file not written in foreground mode (expected before chunk 1 fix)" fi -if [[ -n "$FG_PID" ]] && ps -p "$FG_PID" >/dev/null 2>&1; then +if server_process_running "$FG_PID"; then info "Java process running (pid $FG_PID)" else info "Java PID not available (expected before chunk 1 fix)" @@ -318,7 +349,7 @@ info "Waiting up to ${STARTUP_WAIT}s for server..." if wait_for_server; then info "Server is up" else - info "Server did not respond — continuing" + fail "server did not become ready before the exit propagation test" fi wait_for_pid_file @@ -330,14 +361,14 @@ else fail "bin/pid is empty or missing in foreground mode" fi -if [[ -n "$FG_PID" ]] && ps -p "$FG_PID" >/dev/null 2>&1; then +if server_process_running "$FG_PID"; then pass "Java process running (pid $FG_PID)" else fail "Java process not found for pid '$FG_PID'" fi -if [[ -z "$FG_PID" ]]; then - fail "could not get PID from pid file — skipping exit code check" +if ! server_process_running "$FG_PID"; then + fail "no live Java server — skipping exit code check" kill "$SCRIPT_PID" 2>/dev/null || true else info "Hard-killing Java with SIGKILL (pid $FG_PID)..." @@ -346,15 +377,17 @@ else wait_script_exit "$SCRIPT_PID" ACTUAL_EXIT=$? - if [[ $ACTUAL_EXIT -ne 0 ]]; then + if [[ "$WAIT_TIMED_OUT" == true ]]; then + fail "wrapper timed out after Java SIGKILL — exit code was not propagated" + elif [[ $ACTUAL_EXIT -ne 0 ]]; then pass "script exited non-zero ($ACTUAL_EXIT) after SIGKILL — Docker restart will fire" else fail "script exited 0 after SIGKILL — Docker would NOT restart (exit code lost)" fi - if [[ $ACTUAL_EXIT -eq 137 ]]; then + if [[ "$WAIT_TIMED_OUT" == false && $ACTUAL_EXIT -eq 137 ]]; then pass "exit code is 137 (128+9 for SIGKILL) — correctly propagated" - elif [[ $ACTUAL_EXIT -ne 0 ]]; then + elif [[ "$WAIT_TIMED_OUT" == false && $ACTUAL_EXIT -ne 0 ]]; then info "exit code was $ACTUAL_EXIT (not 137 — may be shell-wrapped, acceptable if non-zero)" fi fi @@ -401,14 +434,14 @@ info "Waiting up to ${STARTUP_WAIT}s for server..." if wait_for_server; then info "Server is up" else - info "Server did not respond — continuing" + fail "server did not become ready before the signal forwarding test" fi wait_for_pid_file FG_PID=$(cat "$PID_FILE" 2>/dev/null || echo "") -if [[ -z "$FG_PID" ]]; then - fail "could not get Java PID — skipping signal forwarding check" +if ! server_process_running "$FG_PID"; then + fail "no live Java server — skipping signal forwarding check" kill "$SCRIPT_PID" 2>/dev/null || true else info "Sending SIGTERM to wrapper script (pid $SCRIPT_PID)..." @@ -426,15 +459,17 @@ else kill "$FG_PID" 2>/dev/null || true fi - if [[ $ACTUAL_EXIT -ne 0 ]]; then + if [[ "$WAIT_TIMED_OUT" == true ]]; then + fail "wrapper timed out after SIGTERM — exit code was not propagated" + elif [[ $ACTUAL_EXIT -ne 0 ]]; then pass "wrapper script exited non-zero ($ACTUAL_EXIT) after SIGTERM" else fail "wrapper script exited 0 after SIGTERM — exit code not propagated" fi - if [[ $ACTUAL_EXIT -eq 143 ]]; then + if [[ "$WAIT_TIMED_OUT" == false && $ACTUAL_EXIT -eq 143 ]]; then pass "exit code is 143 (128+15 for SIGTERM) — correctly propagated" - elif [[ $ACTUAL_EXIT -ne 0 ]]; then + elif [[ "$WAIT_TIMED_OUT" == false && $ACTUAL_EXIT -ne 0 ]]; then info "exit code was $ACTUAL_EXIT (not 143 — may be shell-wrapped, acceptable if non-zero)" fi fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh new file mode 100755 index 0000000000..5d05ab6ef2 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-stop-hugegraph-store.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Verify stop failure propagation and PID retention without signalling real processes. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)" +SCRIPT="$ROOT/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh" +FIXTURE=$(mktemp -d) +trap 'rm -rf "$FIXTURE"' EXIT +mkdir -p "$FIXTURE/bin" +cp "$SCRIPT" "$FIXTURE/bin/stop-hugegraph-store.sh" +cat > "$FIXTURE/bin/util.sh" <<'UTIL' +kill_process_and_wait() { + [[ "$1" == HugeGraphStoreServer && "$2" == 12345 && "$3" == 30 ]] || return 99 + return "$STOP_RESULT" +} +UTIL +printf '12345\n' > "$FIXTURE/bin/pid" +if STOP_RESULT=1 bash "$FIXTURE/bin/stop-hugegraph-store.sh"; then + echo "Stop timeout incorrectly returned success" >&2 + exit 1 +fi +[[ "$(cat "$FIXTURE/bin/pid")" == 12345 ]] +STOP_RESULT=0 bash "$FIXTURE/bin/stop-hugegraph-store.sh" +[[ ! -e "$FIXTURE/bin/pid" ]] +STOP_RESULT=0 bash "$FIXTURE/bin/stop-hugegraph-store.sh" +echo "store-stop-contract-ok" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-distribution.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-distribution.sh new file mode 100755 index 0000000000..ed04b34815 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-distribution.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +COMPONENT="$1" +STANDARD_DIR="$2" +TOPLING_DIR="$3" +TOPLING_TAR="$TOPLING_DIR.tar.gz" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +case "$COMPONENT" in + server) CONFIG_FILE="$TOPLING_DIR/conf/graphs/hugegraph.properties" ;; + pd) CONFIG_FILE="$TOPLING_DIR/conf/application.yml" ;; + store) CONFIG_FILE="$TOPLING_DIR/conf/application-pd.yml" ;; + *) fail "unsupported component: $COMPONENT" ;; +esac + +if [ "${TOPLING_EXPECT_DIRTY_STANDARD:-false}" = "true" ]; then + [ -e "$STANDARD_DIR/bin/pid" ] || + fail "standard distribution has no PID test fixture" + case "$COMPONENT" in + server) + [ -e "$STANDARD_DIR/logs/topling-ci-marker" ] || + fail "standard Server has no logs test fixture" + ;; + pd) + [ -L "$STANDARD_DIR/pd_data" ] || + fail "standard PD has no pd_data symlink test fixture" + ;; + store) + [ -e "$STANDARD_DIR/storage" ] || + fail "standard Store has no storage test fixture" + ;; + esac +fi + +for helper in common-topling.sh prepare-topling.sh preload-topling.sh \ + verify-rocksdb-provider.sh; do + [ -x "$STANDARD_DIR/bin/$helper" ] || + fail "standard distribution is missing helper: $helper" + [ -x "$TOPLING_DIR/bin/$helper" ] || + fail "Topling distribution is missing helper: $helper" +done + +if [ "$COMPONENT" = server ]; then + for distribution in "$STANDARD_DIR" "$TOPLING_DIR"; do + [ -r "$distribution/bin/rocksdb-server-config.sh" ] || + fail "Server distribution is missing rocksdb-server-config.sh" + done +fi + +if find "$STANDARD_DIR/lib" -path '*/topling/*' -print -quit | grep -q .; then + fail "standard distribution contains a Topling JAR" +fi +[ ! -e "$STANDARD_DIR/library/librocksdbjni-linux64.so" ] || + fail "standard distribution contains a Topling native library" + +TOPLING_JAR=$(find "$TOPLING_DIR/lib/topling" -maxdepth 1 \ + -name 'rocksdbjni*.jar' -print -quit) +[ -n "$TOPLING_JAR" ] || fail "Topling distribution contains no Topling JAR" +NATIVE_LIBRARY="$TOPLING_DIR/library/librocksdbjni-linux64.so" +[ -r "$NATIVE_LIBRARY" ] || fail "Topling distribution contains no native library" +[ -r "$TOPLING_DIR/lib/topling/runtime.properties" ] || + fail "Topling distribution contains no runtime marker" +grep -qx 'provider=topling' "$TOPLING_DIR/lib/topling/runtime.properties" || + fail "Topling runtime marker has the wrong provider" +grep -Eq '^[[:space:]]*(rocksdb\.provider=|provider:[[:space:]]*)topling([[:space:]]|$)' \ + "$CONFIG_FILE" || fail "Topling provider is not selected in $CONFIG_FILE" + +for runtime_path in bin/pid logs pd_data rocksdb-data storage; do + if [ -e "$TOPLING_DIR/$runtime_path" ] || + [ -L "$TOPLING_DIR/$runtime_path" ]; then + fail "runtime state leaked into Topling directory: $runtime_path" + fi +done + +[ -r "$TOPLING_TAR" ] || fail "Topling archive not found: $TOPLING_TAR" +ARCHIVE_LIST=$(tar -tzf "$TOPLING_TAR") +if grep -Eq '/(bin/pid|logs|pd_data|rocksdb-data|storage)(/|$)' \ + <<<"$ARCHIVE_LIST"; then + fail "runtime state leaked into Topling archive" +fi +grep -Eq '/lib/topling/runtime.properties$' <<<"$ARCHIVE_LIST" || + fail "Topling archive contains no runtime marker" + +if ldd "$NATIVE_LIBRARY" 2>/dev/null | grep -q 'not found'; then + ldd "$NATIVE_LIBRARY" >&2 || true + fail "Topling native library has unresolved dependencies" +fi + +# Provision the clean package default paths only after checking that neither +# the directory nor archive contains runtime data. The selector seeds markers +# before a subsequent JVM can open these deployment-owned roots. +if [ "$COMPONENT" = server ]; then + mkdir -p "$TOPLING_DIR/rocksdb-data/data" "$TOPLING_DIR/rocksdb-data/wal" +fi +source "$TOPLING_DIR/bin/preload-topling.sh" +[ "$TOPLING_RUNTIME_CLASSPATH" = "$TOPLING_JAR" ] || + fail "Topling classpath does not use the component-local JAR" +[ "$TOPLING_ACTIVE_NATIVE" = "$NATIVE_LIBRARY" ] || + fail "Topling preload does not use the component-local native library" + +echo "PASS: clean component-local Topling distribution: $TOPLING_DIR" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh new file mode 100755 index 0000000000..4c95008cc9 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-docker-entrypoints.sh @@ -0,0 +1,243 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR"/../../../../.. && pwd)" +TEST_ROOT="$(mktemp -d)" + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +prepare_fixture() { + local component="$1" + local entrypoint="$2" + local start_script="$3" + local fixture="$TEST_ROOT/$component" + + mkdir -p "$fixture/bin" + cp "$entrypoint" "$fixture/docker-entrypoint.sh" + cp "$PROJECT_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh" \ + "$fixture/bin/verify-rocksdb-provider.sh" + # shellcheck disable=SC2016 + printf '%s\n' \ + '#!/bin/bash' \ + 'set -euo pipefail' \ + 'printf "%s\n" "$TOPLINGDB_ROCKSDB_PROVIDER" > "$ENTRYPOINT_CAPTURE.provider"' \ + 'printf "%s\n" "$SPRING_APPLICATION_JSON" > "$ENTRYPOINT_CAPTURE.json"' \ + > "$fixture/bin/$start_script" + chmod +x "$fixture/docker-entrypoint.sh" \ + "$fixture/bin/$start_script" \ + "$fixture/bin/verify-rocksdb-provider.sh" +} + +run_pd() { + local provider="$1" + local capture="$TEST_ROOT/pd-$provider" + local data_path="$TEST_ROOT/pd/pd_data" + + if [ "$provider" = "topling" ]; then + data_path="$TEST_ROOT/pd/topling-pd-data" + fi + mkdir -p "$data_path" + ( + cd "$TEST_ROOT/pd" + ENTRYPOINT_CAPTURE="$capture" \ + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER="$provider" \ + HG_PD_ENFORCE_PROVIDER_MARKER=true \ + ./docker-entrypoint.sh >/dev/null + ) + grep -qx "$provider" "$capture.provider" || + fail "PD preload override does not use $provider" + grep -Fq "\"rocksdb\": { \"provider\": \"$provider\" }" \ + "$capture.json" || + fail "PD Spring configuration does not use $provider" + grep -Fq "\"data-path\": \"$data_path\"" "$capture.json" || + fail "PD default data path does not follow $provider" + grep -qx "provider=$provider" \ + "$data_path/.hugegraph-rocksdb-provider" || + fail "PD data path has no $provider marker" +} + +run_store() { + local provider="$1" + local capture="$TEST_ROOT/store-$provider" + local data_path="$TEST_ROOT/store/storage" + + if [ "$provider" = "topling" ]; then + data_path="$TEST_ROOT/store/topling-storage" + fi + mkdir -p "$data_path" + ( + cd "$TEST_ROOT/store" + ENTRYPOINT_CAPTURE="$capture" \ + HG_STORE_PD_ADDRESS=pd:8686 \ + HG_STORE_GRPC_HOST=store \ + HG_STORE_RAFT_ADDRESS=store:8510 \ + HG_STORE_ROCKSDB_PROVIDER="$provider" \ + HG_STORE_ENFORCE_PROVIDER_MARKER=true \ + ./docker-entrypoint.sh >/dev/null + ) + grep -qx "$provider" "$capture.provider" || + fail "Store preload override does not use $provider" + grep -Fq "\"rocksdb\": { \"provider\": \"$provider\" }" \ + "$capture.json" || + fail "Store Spring configuration does not use $provider" + grep -Fq "\"data-path\": \"$data_path\"" "$capture.json" || + fail "Store default data path does not follow $provider" + grep -qx "provider=$provider" \ + "$data_path/.hugegraph-rocksdb-provider" || + fail "Store data path has no $provider marker" +} + +expect_invalid_provider() { + local component="$1" + local output + + if [ "$component" = "pd" ]; then + if output=$( + cd "$TEST_ROOT/pd" + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER=invalid \ + ./docker-entrypoint.sh 2>&1 + ); then + fail "PD accepted an invalid provider" + fi + grep -Fq "HG_PD_ROCKSDB_PROVIDER must be rocksdb or topling" \ + <<<"$output" || fail "PD invalid-provider error is not actionable" + else + if output=$( + cd "$TEST_ROOT/store" + HG_STORE_PD_ADDRESS=pd:8686 \ + HG_STORE_GRPC_HOST=store \ + HG_STORE_RAFT_ADDRESS=store:8510 \ + HG_STORE_ROCKSDB_PROVIDER=invalid \ + ./docker-entrypoint.sh 2>&1 + ); then + fail "Store accepted an invalid provider" + fi + grep -Fq "HG_STORE_ROCKSDB_PROVIDER must be rocksdb or topling" \ + <<<"$output" || fail "Store invalid-provider error is not actionable" + fi +} + +prepare_fixture \ + pd \ + "$PROJECT_ROOT/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh" \ + start-hugegraph-pd.sh +prepare_fixture \ + store \ + "$PROJECT_ROOT/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh" \ + start-hugegraph-store.sh + +for provider in rocksdb topling; do + run_pd "$provider" + run_store "$provider" +done +expect_invalid_provider pd +expect_invalid_provider store + +SHARED_DATA="$TEST_ROOT/shared-data" +mkdir -p "$SHARED_DATA" +( + cd "$TEST_ROOT/pd" + ENTRYPOINT_CAPTURE="$TEST_ROOT/pd-shared-topling" \ + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER=topling \ + HG_PD_DATA_PATH="$SHARED_DATA" \ + ./docker-entrypoint.sh >/dev/null +) +if output=$( + cd "$TEST_ROOT/pd" + ENTRYPOINT_CAPTURE="$TEST_ROOT/pd-shared-rocksdb" \ + HG_PD_AUTH_SECRET_KEY=ci-topling-pd-secret \ + HG_PD_GRPC_HOST=pd \ + HG_PD_RAFT_ADDRESS=pd:8610 \ + HG_PD_RAFT_PEERS_LIST=pd:8610 \ + HG_PD_INITIAL_STORE_LIST=store:8500 \ + HG_PD_ROCKSDB_PROVIDER=rocksdb \ + HG_PD_DATA_PATH="$SHARED_DATA" \ + ./docker-entrypoint.sh 2>&1 +); then + fail "PD accepted a Topling data path for standard RocksDB" +fi +grep -Fq "provider marker mismatch" <<<"$output" || + fail "PD provider mismatch error is not actionable" +grep -qx "provider=topling" \ + "$SHARED_DATA/.hugegraph-rocksdb-provider" || + fail "provider mismatch changed the existing data marker" + +NONEMPTY_DATA="$TEST_ROOT/nonempty-data" +mkdir -p "$NONEMPTY_DATA" +touch "$NONEMPTY_DATA/existing.sst" +PROVIDER_HELPER="$PROJECT_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh" +if "$PROVIDER_HELPER" \ + store topling "$NONEMPTY_DATA" false >/dev/null 2>&1; then + fail "ToplingDB accepted a non-empty unmarked data path" +fi +[ -f "$NONEMPTY_DATA/existing.sst" ] || + fail "failed provider validation removed existing data" +[ ! -e "$NONEMPTY_DATA/.hugegraph-rocksdb-provider" ] || + fail "failed provider validation created a marker" + +OUTSIDE_DATA="$TEST_ROOT/outside-data" +mkdir -p "$OUTSIDE_DATA" +ln -s "$OUTSIDE_DATA" "$TEST_ROOT/data-link" +if "$PROVIDER_HELPER" \ + server topling "$TEST_ROOT/data-link/new" true >/dev/null 2>&1; then + fail "ToplingDB accepted a data path with a symlink prefix" +fi +[ ! -e "$OUTSIDE_DATA/new" ] || + fail "provider validation created a directory through a symlink prefix" + +CONCURRENT_DATA="$TEST_ROOT/concurrent-data" +mkdir -p "$CONCURRENT_DATA" +"$PROVIDER_HELPER" server topling "$CONCURRENT_DATA" true >/dev/null & +FIRST_PID=$! +"$PROVIDER_HELPER" server topling "$CONCURRENT_DATA" true >/dev/null & +SECOND_PID=$! +wait "$FIRST_PID" || + fail "first concurrent provider-marker initialization failed" +wait "$SECOND_PID" || + fail "second concurrent provider-marker initialization failed" +grep -qx "provider=topling" \ + "$CONCURRENT_DATA/.hugegraph-rocksdb-provider" || + fail "concurrent provider-marker initialization wrote the wrong marker" + +echo "PASS: PD and Store Docker entrypoints keep Spring and preload providers aligned" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-native-diagnostic.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-native-diagnostic.sh new file mode 100755 index 0000000000..9f9d0d5fc1 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-native-diagnostic.sh @@ -0,0 +1,158 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_ROOT=$(mktemp -d) +trap 'rm -rf "$TEST_ROOT"' EXIT +WRAPPER="${1:-$SCRIPT_DIR/run-topling-native-diagnostic.sh}" + +cat > "$TEST_ROOT/probe.sh" <<'FIXTURE' +#!/bin/bash +set -eu +mode="$1" +echo 'JNI identity fixture: sha256=fixture-only' +if [ "$mode" = probe ]; then + case "$CASE" in + prereq-uppercase) + echo 'ERROR: unexpected native initialization failure' + echo 'Topling diagnostic phase: complete' + exit 0 ;; + optional-warning) + echo 'WARN: access java.nio.DirectByteBuffer failed: java.lang.NoSuchMethodException: no such constructor: java.nio.DirectByteBuffer.(long,long)void/newInvokeSpecial' ;; + prereq-zero) + echo 'Topling diagnostic phase: complete' + echo 'native.cc:1: unknown: verify(ok) failed: db not closed' + exit 0 ;; + prereq-incomplete) + exit 0 ;; + prereq-*) + echo "${CASE#prereq-}" + exit 1 ;; + esac + echo 'Topling diagnostic phase: runtime-check' + echo 'Topling diagnostic phase: complete' + echo 'Runtime smoke test passed: provider=topling, mode=probe' + exit 0 +fi +case "$CASE" in + pass|optional-warning) + echo 'Topling diagnostic phase: complete' + exit 0 ;; + uppercase-error|lowercase-error|bracket-error) + echo 'Topling diagnostic phase: cf-lifecycle' + echo 'side_plugin_repo.cc:123: CloseAllDB: verify(iter != end) failed: cfh must in cfh_to_view !' + case "$CASE" in + uppercase-error) echo 'ERROR: unexpected native failure' ;; + lowercase-error) echo 'error: unexpected native failure' ;; + *) echo '[ERROR] unexpected native failure' ;; + esac + exit 134 ;; + mixed|java-error|unknown-assertion|zero-unknown) + echo 'Topling diagnostic phase: cf-lifecycle' + if [ "$CASE" = mixed ] || [ "$CASE" = java-error ]; then + echo 'side_plugin_repo.cc:123: CloseAllDB: verify(iter != end) failed: cfh must in cfh_to_view !' + fi + if [ "$CASE" = java-error ]; then + echo 'java.lang.IllegalStateException: create CF failed' + else + echo 'native.cc:1: unknown: verify(ok) failed: db not closed' + fi + if [ "$CASE" = zero-unknown ]; then + echo 'Topling diagnostic phase: complete' + exit 0 + fi + exit 134 ;; + known|known-cpp|wrong-status|early|silent) + if [ "$CASE" != early ]; then + echo 'Topling diagnostic phase: cf-lifecycle' + fi + if [ "$CASE" = known-cpp ]; then + echo 'sideplugin/rockside/src/topling/builtin_plugin_misc.cc:4596: rocksdb::SidePluginRepo::CloseOneDB(rocksdb::DB*, bool)::: verify(cfh_to_view.end() != iter) failed: cfh must in cfh_to_view !' + else + echo 'side_plugin_repo.cc:123: CloseAllDB: verify(iter != end) failed: cfh must in cfh_to_view !' + fi + case "$CASE" in + wrong-status) exit 1 ;; + silent) exit 0 ;; + *) exit 134 ;; + esac ;; + *) + echo 'Topling diagnostic phase: runtime-check' + echo "$CASE" + # A secondary native abort during cleanup must not hide this failure. + echo 'side_plugin_repo.cc:123: CloseAllDB: verify(iter != end) failed: cfh must in cfh_to_view !' + exit 134 ;; +esac +FIXTURE +chmod +x "$TEST_ROOT/probe.sh" + +run_case() { + local name="$1" expected="$2" classification="$3" status=0 + local case_dir="$TEST_ROOT/${1//\//_}" + CASE="$name" GITHUB_STEP_SUMMARY="$case_dir.summary" bash "$WRAPPER" "$case_dir" "$TEST_ROOT/probe.sh" > "$case_dir.output" 2>&1 || status=$? + if [ "$status" -ne "$expected" ]; then + cat "$case_dir.output" + echo "FAIL: $name expected status $expected, got $status" >&2 + exit 1 + fi + grep -Fq "Result: $classification" "$case_dir/report.md" + cmp "$case_dir/report.md" "$case_dir.summary" + if [ "$classification" != known-cf-assertion ] && + grep -Fq '#212' "$case_dir/report.md"; then + echo "FAIL: $name was incorrectly attributed to #212" >&2 + exit 1 + fi + grep -Fq 'JNI identity fixture:' "$case_dir/probe.log" + echo "PASS: $name" +} + +run_case pass 0 passed +run_case known 0 known-cf-assertion +run_case known-cpp 0 known-cf-assertion +run_case optional-warning 0 passed +run_case prereq-uppercase 1 prerequisite-failed +for failure in uppercase-error lowercase-error bracket-error wrong-status early silent mixed java-error unknown-assertion zero-unknown \ + 'Error: missing config' \ + 'java.lang.NoClassDefFoundError: org/rocksdb/RocksDB' \ + 'java.lang.UnsatisfiedLinkError: missing JNI' \ + 'Expected native library is not mapped' \ + 'initial read returned unexpected data'; do + run_case "$failure" 1 unknown-failure +done + +for failure in \ + 'Error: readable ToplingDB Easy Migrate config is required' \ + 'java.lang.NoClassDefFoundError: org/rocksdb/RocksDB' \ + 'java.lang.UnsatisfiedLinkError: missing JNI' \ + 'Expected native library is not mapped' \ + 'probe read returned unexpected data'; do + run_case "prereq-$failure" 1 prerequisite-failed +done + +run_case prereq-zero 1 prerequisite-failed +run_case prereq-incomplete 1 prerequisite-failed + +# Compile against the selected API without loading native code, then verify that +# a Java close failure is visible before any subsequent native cleanup can abort. +JNI_JAR="$SCRIPT_DIR/../static/lib/topling/rocksdbjni-8.10.2-20260725.141011-1.jar" +javac -cp "$JNI_JAR" -d "$TEST_ROOT/classes" \ + "$SCRIPT_DIR/RocksDBRuntimeSmokeTest.java" \ + "$SCRIPT_DIR/ToplingDiagnosticResourcesTest.java" +java -cp "$TEST_ROOT/classes:$JNI_JAR" ToplingDiagnosticResourcesTest diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-provider-ownership.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-provider-ownership.sh new file mode 100755 index 0000000000..f403bdd320 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-provider-ownership.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Portable standard ownership, plus real-helper concurrency when run on Linux. +set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONFIG_HELPER="$(cd "$SCRIPT_DIR/../static/bin" && pwd)/rocksdb-server-config.sh" +MARKER_HELPER="$(dirname "$CONFIG_HELPER")/verify-rocksdb-provider.sh" +TEST_ROOT=$(mktemp -d) +TEST_ROOT=$(cd -P "$TEST_ROOT" && pwd) +standard_pid="" +topling_pid="" +cleanup() { + local pid + for pid in "$standard_pid" "$topling_pid"; do + [ -n "$pid" ] || continue + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + done + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT +# shellcheck source=/dev/null +source "$CONFIG_HELPER" +fail() { echo "FAIL: $*" >&2; exit 1; } +claim_standard() { server_verify_storage_path "$TEST_ROOT" rocksdb "$1" false; } + +claim_standard "$TEST_ROOT/new/data" +grep -qx provider=rocksdb "$TEST_ROOT/new/data/.hugegraph-rocksdb-provider" +echo "PASS: portable standard startup atomically claims new roots" +mkdir -p "$TEST_ROOT/legacy" +printf '%s\n' existing-data > "$TEST_ROOT/legacy/SENTINEL" +claim_standard "$TEST_ROOT/legacy" +[ ! -e "$TEST_ROOT/legacy/.hugegraph-rocksdb-provider" ] || fail "legacy root was relabeled" +grep -qx existing-data "$TEST_ROOT/legacy/SENTINEL" +echo "PASS: nonempty unmarked standard legacy data remains unchanged" + +mkdir -p "$TEST_ROOT/foreign" +printf '%s\n' format=1 component=server provider=topling > "$TEST_ROOT/foreign/.hugegraph-rocksdb-provider" +cp "$TEST_ROOT/foreign/.hugegraph-rocksdb-provider" "$TEST_ROOT/foreign-before" +if claim_standard "$TEST_ROOT/foreign/missing" > "$TEST_ROOT/output" 2>&1; then fail "foreign ancestor accepted"; fi +[ ! -e "$TEST_ROOT/foreign/missing" ] || fail "foreign ancestor was modified" +cmp "$TEST_ROOT/foreign-before" "$TEST_ROOT/foreign/.hugegraph-rocksdb-provider" +echo "PASS: conflicting ancestor rejected before directory creation" + +mkdir -p "$TEST_ROOT/pending" +touch "$TEST_ROOT/pending/.provider-marker.pending" +if claim_standard "$TEST_ROOT/pending" > "$TEST_ROOT/output" 2>&1; then fail "in-progress ownership accepted"; fi +grep -Fq 'initialization is in progress' "$TEST_ROOT/output" +[ ! -e "$TEST_ROOT/pending/.hugegraph-rocksdb-provider" ] || fail "pending ownership was replaced" +echo "PASS: in-progress Topling marker cannot be treated as legacy data" + +mkdir -p "$TEST_ROOT/partial" +touch "$TEST_ROOT/partial/.hugegraph-rocksdb-provider" +if claim_standard "$TEST_ROOT/partial" > "$TEST_ROOT/output" 2>&1; then fail "partial marker accepted"; fi +[ ! -s "$TEST_ROOT/partial/.hugegraph-rocksdb-provider" ] || fail "partial marker was overwritten" +echo "PASS: interrupted standard marker stays fail-closed" + +# Pause immediately after an empty-directory scan, before either provider can +# publish ownership. The barrier is outside the storage root being checked. +mkdir -p "$TEST_ROOT/fake-bin" "$TEST_ROOT/barrier" +REAL_FIND=$(command -v find) +export REAL_FIND +cat > "$TEST_ROOT/fake-bin/find" <<'SH' +#!/bin/bash +status=0 +"$REAL_FIND" "$@" || status=$? +case " $* " in + *' lost+found '*) + touch "$BARRIER_DIR/$OWNER_ACTOR.ready" + while [ ! -f "$BARRIER_DIR/release" ]; do sleep 0.01; done + ;; +esac +exit "$status" +SH +chmod +x "$TEST_ROOT/fake-bin/find" +wait_ready() { + local actor="$1" attempt + for ((attempt = 0; attempt < 1000; attempt++)); do + [ ! -f "$TEST_ROOT/barrier/$actor.ready" ] || return 0 + sleep 0.01 + done + fail "$actor did not reach ownership barrier" +} +run_standard_actor() { + PATH="$TEST_ROOT/fake-bin:$PATH" OWNER_ACTOR=standard BARRIER_DIR="$TEST_ROOT/barrier" \ + bash -c 'source "$1"; server_verify_storage_path "$2" rocksdb "$3" false' \ + _ "$CONFIG_HELPER" "$TEST_ROOT" "$TEST_ROOT/race" +} +mkdir "$TEST_ROOT/race" +run_standard_actor > "$TEST_ROOT/standard.log" 2>&1 & +standard_pid=$! +wait_ready standard +printf '%s\n' format=1 component=server provider=topling > "$TEST_ROOT/race/.provider-marker.fixture" +mv -n "$TEST_ROOT/race/.provider-marker.fixture" "$TEST_ROOT/race/.hugegraph-rocksdb-provider" +touch "$TEST_ROOT/barrier/release" +standard_status=0 +wait "$standard_pid" || standard_status=$? +standard_pid="" +[ "$standard_status" != 0 ] || fail "standard opened after Topling won the empty-root race" +grep -qx provider=topling "$TEST_ROOT/race/.hugegraph-rocksdb-provider" +echo "PASS: standard loses cleanly when Topling claims after its empty scan" + +if [ "$(uname -s)" = Linux ]; then + rm -rf "$TEST_ROOT/race" "$TEST_ROOT/barrier" + mkdir "$TEST_ROOT/race" "$TEST_ROOT/barrier" + run_standard_actor > "$TEST_ROOT/standard.log" 2>&1 & + standard_pid=$! + PATH="$TEST_ROOT/fake-bin:$PATH" OWNER_ACTOR=topling BARRIER_DIR="$TEST_ROOT/barrier" \ + bash "$MARKER_HELPER" server topling "$TEST_ROOT/race" true > "$TEST_ROOT/topling.log" 2>&1 & + topling_pid=$! + wait_ready standard + wait_ready topling + touch "$TEST_ROOT/barrier/release" + standard_status=0 + topling_status=0 + wait "$standard_pid" || standard_status=$? + standard_pid="" + wait "$topling_pid" || topling_status=$? + topling_pid="" + if [ "$standard_status" = 0 ] && [ "$topling_status" = 0 ]; then fail "both providers acquired the empty root"; fi + if [ "$standard_status" != 0 ] && [ "$topling_status" != 0 ]; then + cat "$TEST_ROOT/standard.log" "$TEST_ROOT/topling.log" + fail "neither provider acquired the empty root" + fi + if [ "$standard_status" = 0 ]; then provider=rocksdb; else provider=topling; fi + grep -qx "provider=$provider" "$TEST_ROOT/race/.hugegraph-rocksdb-provider" + echo "PASS: simultaneous standard and real Topling helper admit exactly one provider" + + # Docker owns a deployment parent; bare launch owns configured children. + # A parent claim started first must notice a child created after its scan. + rm -rf "$TEST_ROOT/race" "$TEST_ROOT/barrier" + mkdir "$TEST_ROOT/race" "$TEST_ROOT/barrier" + PATH="$TEST_ROOT/fake-bin:$PATH" OWNER_ACTOR=topling BARRIER_DIR="$TEST_ROOT/barrier" \ + bash "$MARKER_HELPER" server topling "$TEST_ROOT/race" true > "$TEST_ROOT/topling.log" 2>&1 & + topling_pid=$! + wait_ready topling + claim_standard "$TEST_ROOT/race/data" + touch "$TEST_ROOT/barrier/release" + topling_status=0 + wait "$topling_pid" || topling_status=$? + topling_pid="" + [ "$topling_status" != 0 ] || fail "Topling parent hid an already claimed standard child" + [ ! -e "$TEST_ROOT/race/.hugegraph-rocksdb-provider" ] || fail "conflicting parent was marked" + grep -qx provider=rocksdb "$TEST_ROOT/race/data/.hugegraph-rocksdb-provider" + grep -Fq 'data path changed while initializing provider marker' "$TEST_ROOT/topling.log" + echo "PASS: Topling deployment parent cannot override a newly claimed standard child" +else + echo "SKIP: real Topling helper concurrency requires Linux" +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-packaging.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-packaging.sh new file mode 100755 index 0000000000..bc921463d2 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-packaging.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +if [ "$#" -ne 3 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +COMPONENT_DIRS=("$1" "$2" "$3") +TEST_ROOT="$(mktemp -d)" +FAKE_BIN="$TEST_ROOT/fake-bin" + +mkdir -p "$FAKE_BIN" +printf '%s\n' \ + '#!/bin/sh' \ + 'case "${1:-}" in' \ + ' -s) echo Linux ;;' \ + ' -m) echo x86_64 ;;' \ + ' *) echo Linux ;;' \ + 'esac' > "$FAKE_BIN/uname" +printf '%s\n' '#!/bin/sh' 'echo "all dependencies resolved"' > "$FAKE_BIN/ldd" +chmod +x "$FAKE_BIN/uname" "$FAKE_BIN/ldd" + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +reference_checksum="" +for component_dir in "${COMPONENT_DIRS[@]}"; do + for helper in common-topling.sh prepare-topling.sh preload-topling.sh \ + verify-rocksdb-provider.sh; do + if [ ! -x "$component_dir/bin/$helper" ]; then + fail "missing executable helper: $component_dir/bin/$helper" + fi + done + + if find "$component_dir/lib" -path '*/topling/*' -print -quit | grep -q .; then + fail "standard artifact contains a Topling JAR: $component_dir" + fi + if [ -e "$component_dir/library/librocksdbjni-linux64.so" ]; then + fail "standard artifact contains a Topling native library: $component_dir" + fi + + env -u TOPLINGDB_EASY_MIGRATE_CONF PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1/bin/preload-topling.sh" + test -z "${TOPLINGDB_EASY_MIGRATE_CONF:-}" + test -z "${TOPLING_RUNTIME_CLASSPATH:-}" + ' _ "$component_dir" + + checksum=$(sha256sum "$component_dir/bin/preload-topling.sh" | + awk '{ print $1 }') + if [ -z "$reference_checksum" ]; then + reference_checksum="$checksum" + elif [ "$checksum" != "$reference_checksum" ]; then + fail "packaged runtime selectors differ across components" + fi + + fixture="$TEST_ROOT/$(basename "$component_dir")" + cp -R "$component_dir" "$fixture" + if TOPLINGDB_ROCKSDB_PROVIDER=topling \ + PATH="$FAKE_BIN:$PATH" bash -c \ + 'source "$1/bin/preload-topling.sh"' _ "$fixture" \ + > "$fixture/preload.out" 2>&1; then + fail "standard artifact accepted Topling without a local runtime: $component_dir" + fi + expected_error="no prepared ToplingDB JAR found in $fixture/lib/topling/" + if [ -f "$fixture/conf/rest-server.properties" ]; then + [ -r "$fixture/bin/rocksdb-server-config.sh" ] || fail "missing Server config helper" + expected_error="conflicts with configured rocksdb.provider" + fi + if ! grep -Fq "$expected_error" "$fixture/preload.out"; then + sed -n '1,120p' "$fixture/preload.out" >&2 + fail "missing component-local runtime error: $component_dir" + fi + + echo "PASS: standard artifact is isolated and fail-fast: $component_dir" +done diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh new file mode 100755 index 0000000000..69ce616cd4 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-runtime-selection.sh @@ -0,0 +1,368 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DIST_ROOT="$(cd "$SCRIPT_DIR"/../../.. && pwd)" +PRELOAD_SOURCE="$DIST_ROOT/src/assembly/static/bin/preload-topling.sh" +TEST_ROOT="$(mktemp -d)" +COMPONENT_ROOT="$TEST_ROOT/component" +FAKE_BIN="$TEST_ROOT/fake-bin" + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +expect_failure() { + local name="$1" + local expected="$2" + shift 2 + local output + + if output=$("$@" 2>&1); then + fail "$name unexpectedly succeeded" + fi + if ! grep -Fq "$expected" <<<"$output"; then + echo "$output" >&2 + fail "$name did not report: $expected" + fi + echo "PASS: $name" +} + +reset_fixture() { + rm -rf "$COMPONENT_ROOT" "$FAKE_BIN" + mkdir -p "$COMPONENT_ROOT/bin" "$COMPONENT_ROOT/conf/graphs" \ + "$COMPONENT_ROOT/lib/topling" "$FAKE_BIN" + cp "$PRELOAD_SOURCE" "$COMPONENT_ROOT/bin/preload-topling.sh" + cp "$(dirname "$PRELOAD_SOURCE")/rocksdb-server-config.sh" "$COMPONENT_ROOT/bin/" + printf '%s\n' 'graphs=./conf/graphs' > "$COMPONENT_ROOT/conf/rest-server.properties" + + printf '%s\n' \ + '#!/bin/sh' \ + 'case "${1:-}" in' \ + ' -s) echo Linux ;;' \ + ' -m) echo x86_64 ;;' \ + ' *) echo Linux ;;' \ + 'esac' > "$FAKE_BIN/uname" + printf '%s\n' '#!/bin/sh' 'echo "all dependencies resolved"' > "$FAKE_BIN/ldd" + # This fixture tests selection, not filesystem ownership. Linux path and + # marker operations have a separate test using the real helper in Linux. + printf '%s\n' '#!/bin/bash' 'exit 0' > "$COMPONENT_ROOT/bin/verify-rocksdb-provider.sh" + chmod +x "$COMPONENT_ROOT/bin/verify-rocksdb-provider.sh" + chmod +x "$FAKE_BIN/uname" "$FAKE_BIN/ldd" +} + +source_preload() { + PATH="$FAKE_BIN:$PATH" bash -c 'source "$1"' _ \ + "$COMPONENT_ROOT/bin/preload-topling.sh" +} + +source_preload_override() { + TOPLINGDB_ROCKSDB_PROVIDER="$1" PATH="$FAKE_BIN:$PATH" \ + bash -c 'source "$1"' _ \ + "$COMPONENT_ROOT/bin/preload-topling.sh" +} + +reset_fixture +source_preload +echo "PASS: unset provider selects standard RocksDB without Topling runtime" + +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=not-topling' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +expect_failure "invalid provider" \ + "invalid rocksdb.provider 'not-topling'" \ + source_preload + +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=rocksdb' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +expect_failure "invalid provider override" \ + "invalid TOPLINGDB_ROCKSDB_PROVIDER 'not-topling'" \ + source_preload_override not-topling + +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=rocksdb' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/conf/graphs/second.properties" +expect_failure "conflicting providers" \ + "conflicting rocksdb.provider values" \ + source_preload + +rm -f "$COMPONENT_ROOT/conf/graphs/second.properties" +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +mkdir -p "$TEST_ROOT/apache-hugegraph-server/lib/topling" +touch "$TEST_ROOT/apache-hugegraph-server/lib/topling/rocksdbjni-stray.jar" +expect_failure "component-local JAR is required" \ + "no prepared ToplingDB JAR found in $COMPONENT_ROOT/lib/topling/" \ + source_preload + +touch "$COMPONENT_ROOT/lib/topling/rocksdbjni-topling.jar" +expect_failure "Easy Migrate configuration is required" \ + "required ToplingDB Easy Migrate config not found" \ + source_preload + +printf '%s\n' 'http:' ' auto_start_http: false' \ + > "$COMPONENT_ROOT/conf/toplingdb.yaml" +expect_failure "component-local native library is required" \ + "prepared ToplingDB native library not found" \ + source_preload + +mkdir -p "$COMPONENT_ROOT/library" +touch "$COMPONENT_ROOT/library/librocksdbjni-linux64.so" +printf '%s\n' '#!/bin/sh' 'echo "libmissing.so => not found"' > "$FAKE_BIN/ldd" +chmod +x "$FAKE_BIN/ldd" +expect_failure "unresolved native dependency" \ + "native library has unresolved system dependencies" \ + source_preload + +printf '%s\n' '#!/bin/sh' 'echo "all dependencies resolved"' > "$FAKE_BIN/ldd" +chmod +x "$FAKE_BIN/ldd" +PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1" + test "$TOPLING_RUNTIME_CLASSPATH" = "$2/lib/topling/rocksdbjni-topling.jar" + test "$TOPLINGDB_EASY_MIGRATE_CONF" = "$2/conf/toplingdb.yaml" + test "$TOPLING_ACTIVE_NATIVE" = "$2/library/librocksdbjni-linux64.so" +' _ "$COMPONENT_ROOT/bin/preload-topling.sh" "$COMPONENT_ROOT" +echo "PASS: valid component-local Topling runtime is selected" + +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=rocksdb' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +expect_failure "override cannot change Java rocksdb provider" \ + "conflicts with configured rocksdb.provider" \ + source_preload_override topling +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +expect_failure "override cannot change Java topling provider" \ + "conflicts with configured rocksdb.provider" \ + source_preload_override rocksdb + +printf '%s\n' 'backend=ROCKSDB' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1" + test "$PROVIDER" = topling +' _ "$COMPONENT_ROOT/bin/preload-topling.sh" +echo "PASS: uppercase RocksDB backend selects its configured JNI provider" + +# Unspecified providers inherit Java's rocksdb default, even alongside TP. +printf '%s\n' 'backend=rocksdb' > "$COMPONENT_ROOT/conf/graphs/implicit.properties" +expect_failure "implicit default conflicts with explicit Topling" \ + "conflicting rocksdb.provider values" source_preload +rm "$COMPONENT_ROOT/conf/graphs/implicit.properties" +printf '%s\n' 'backend=hstore' 'rocksdb.provider=rocksdb' \ + > "$COMPONENT_ROOT/conf/graphs/remote.properties" +source_preload +echo "PASS: non-local backend does not select JNI" + +mkdir -p "$COMPONENT_ROOT/custom graphs" +mv "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" "$COMPONENT_ROOT/custom graphs/" +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=rocksdb' \ + > "$COMPONENT_ROOT/conf/graphs/hugegraph.properties" +for graphs in './custom graphs' "$COMPONENT_ROOT/custom graphs" './custom\ graphs'; do + printf 'graphs=%s\n' "$graphs" > "$COMPONENT_ROOT/conf/rest-server.properties" + PATH="$FAKE_BIN:$PATH" bash -c ' + source "$1" + test "$PROVIDER" = topling + ' _ "$COMPONENT_ROOT/bin/preload-topling.sh" +done +echo "PASS: custom relative, absolute and escaped-space graph directories" +printf '%s\n' 'graphs=./missing' > "$COMPONENT_ROOT/conf/rest-server.properties" +expect_failure "missing graphs directory" "graphs directory does not exist" source_preload +printf '%s\n' 'graphs=./custom graphs' > "$COMPONENT_ROOT/conf/rest-server.properties" +expect_failure "custom graphs override conflict" \ + "conflicts with configured rocksdb.provider" source_preload_override rocksdb +printf '%s\n' 'rocksdb.provider=rocksdb' >> "$COMPONENT_ROOT/custom graphs/hugegraph.properties" +expect_failure "duplicate provider is rejected" "duplicate property" source_preload +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=${runtime}' \ + > "$COMPONENT_ROOT/custom graphs/hugegraph.properties" +expect_failure "interpolated provider is rejected" "interpolation/list syntax is unsupported" source_preload + +# A Topling distribution must also be able to select standard RocksDB without +# putting its optional runtime JAR on the HugeGraph Server classpath. Exercise +# the launcher with a fake JVM and inspect the actual -cp argument rather than +# relying on a source-text assertion. +SERVER_LAUNCHER_ROOT="$TEST_ROOT/server launcher" +SERVER_JAVA_CAPTURE="$TEST_ROOT/server-java-classpath" +mkdir -p "$SERVER_LAUNCHER_ROOT/bin" "$SERVER_LAUNCHER_ROOT/conf" \ + "$SERVER_LAUNCHER_ROOT/lib/topling" "$SERVER_LAUNCHER_ROOT/ext" \ + "$SERVER_LAUNCHER_ROOT/plugins" "$SERVER_LAUNCHER_ROOT/logs" +mkdir -p "$SERVER_LAUNCHER_ROOT/conf/graphs" +cp "$(dirname "$PRELOAD_SOURCE")/rocksdb-server-config.sh" "$SERVER_LAUNCHER_ROOT/bin/" +mkdir -p "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow/classes" +mkdir -p "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow.jar" +cp "$DIST_ROOT/src/assembly/static/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" +cp "$DIST_ROOT/src/assembly/static/bin/init-store.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" +cp "$DIST_ROOT/src/assembly/static/bin/preload-topling.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/preload-topling.sh" +cp "$DIST_ROOT/src/assembly/static/bin/util.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/util.sh" +touch "$SERVER_LAUNCHER_ROOT/lib/hugegraph-server-bootstrap.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling/log4j-slf4j-impl-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling/hugegraph-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling/rocksdbjni-topling.jar" +ln -s "$SERVER_LAUNCHER_ROOT/lib/topling/log4j-slf4j-impl-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/log4j-slf4j-impl-topling-alias.jar" +ln -s "$SERVER_LAUNCHER_ROOT/lib/topling/hugegraph-topling.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-topling-alias.jar" +ln -s "$SERVER_LAUNCHER_ROOT/lib/topling" \ + "$SERVER_LAUNCHER_ROOT/lib/topling-alias" +printf '%s\n' \ + 'gremlinserver.url=http://127.0.0.1:43123' \ + 'restserver.url=http://127.0.0.1:43124' \ + > "$SERVER_LAUNCHER_ROOT/conf/rest-server.properties" +printf '%s\n' \ + '#!/bin/sh' \ + 'if [ "${1:-}" = "-server" ]; then shift; fi' \ + 'if [ "${1:-}" = "-version" ]; then' \ + ' echo '\''openjdk version "11.0.32"'\'' >&2' \ + ' exit 0' \ + 'fi' \ + 'while [ "$#" -gt 0 ]; do' \ + ' if [ "$1" = "-cp" ]; then' \ + ' printf "%s\\n" "$2" > "$JAVA_CAPTURE"' \ + ' exit 0' \ + ' fi' \ + ' shift' \ + 'done' \ + 'exit 0' \ + > "$FAKE_BIN/java" +chmod +x "$FAKE_BIN/java" "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" \ + "$SERVER_LAUNCHER_ROOT/bin/preload-topling.sh" +JAVA_HOME='' JAVA_OPTIONS=-Xmx64m STDOUT_MODE=true CLASSPATH='' \ + JAVA_CAPTURE="$SERVER_JAVA_CAPTURE" PATH="$FAKE_BIN:$PATH" \ + bash "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/conf/gremlin-server.yaml" \ + "$SERVER_LAUNCHER_ROOT/conf/rest-server.properties" true \ + >/dev/null 2>&1 +grep -Fq "$SERVER_LAUNCHER_ROOT/lib/hugegraph-server-bootstrap.jar" \ + "$SERVER_JAVA_CAPTURE" || + fail "Server launcher omitted the standard bootstrap JAR" +if grep -Fq "$SERVER_LAUNCHER_ROOT/lib/topling/" "$SERVER_JAVA_CAPTURE"; then + fail "standard provider launcher leaked the optional Topling classpath" +fi +for leaked in \ + "$SERVER_LAUNCHER_ROOT/lib/log4j-slf4j-impl-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling-alias/" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow"; do + if grep -Fq "$leaked" "$SERVER_JAVA_CAPTURE"; then + fail "standard provider launcher followed a Topling symlink: $leaked" + fi +done +echo "PASS: standard provider launcher excludes Topling JARs" + +SERVER_INIT_CAPTURE="$TEST_ROOT/server-init-classpath" +JAVA_HOME='' CLASSPATH='' JAVA_CAPTURE="$SERVER_INIT_CAPTURE" \ + PATH="$FAKE_BIN:$PATH" bash "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" \ + >/dev/null 2>&1 +grep -Fq "$SERVER_LAUNCHER_ROOT/lib/hugegraph-server-bootstrap.jar" \ + "$SERVER_INIT_CAPTURE" || + fail "init-store omitted the standard bootstrap JAR" +if grep -Fq "$SERVER_LAUNCHER_ROOT/lib/topling/" "$SERVER_INIT_CAPTURE"; then + fail "standard provider init-store leaked the optional Topling classpath" +fi +for leaked in \ + "$SERVER_LAUNCHER_ROOT/lib/log4j-slf4j-impl-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-topling-alias.jar" \ + "$SERVER_LAUNCHER_ROOT/lib/topling-alias/" \ + "$SERVER_LAUNCHER_ROOT/lib/hugegraph-shadow"; do + if grep -Fq "$leaked" "$SERVER_INIT_CAPTURE"; then + fail "standard provider init-store followed a Topling symlink: $leaked" + fi +done +echo "PASS: standard provider init-store excludes Topling JARs" + +# The low-level launcher must honor its REST argument, not the default file. +mkdir -p "$SERVER_LAUNCHER_ROOT/selected graphs" +printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=topling' \ + > "$SERVER_LAUNCHER_ROOT/selected graphs/hugegraph.properties" +printf '%s\n' 'graphs=./selected graphs' > "$SERVER_LAUNCHER_ROOT/conf/selected rest.properties" +rm "$SERVER_JAVA_CAPTURE" +expect_failure "direct launcher resolves the supplied REST config" \ + "conflicts with configured rocksdb.provider" \ + env JAVA_HOME='' JAVA_OPTIONS=-Xmx64m STDOUT_MODE=true CLASSPATH='' \ + TOPLINGDB_ROCKSDB_PROVIDER=rocksdb JAVA_CAPTURE="$SERVER_JAVA_CAPTURE" PATH="$FAKE_BIN:$PATH" \ + bash "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/conf/gremlin-server.yaml" 'conf/selected rest.properties' true +[ ! -e "$SERVER_JAVA_CAPTURE" ] || fail "conflicting config reached the JVM" +printf '%s\n' 'graphs=./selected graphs' > "$SERVER_LAUNCHER_ROOT/conf/rest-server.properties" +expect_failure "init-store resolves the same graph directory" \ + "conflicts with configured rocksdb.provider" \ + env JAVA_HOME='' TOPLINGDB_ROCKSDB_PROVIDER=rocksdb PATH="$FAKE_BIN:$PATH" \ + bash "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" + +printf '%s\n' 'backend=rocksdb' 'unrelated=value\' 'rocksdb.provider=topling' \ + > "$COMPONENT_ROOT/custom graphs/hugegraph.properties" +expect_failure "unrelated continuation cannot hide provider" \ + "property continuations are unsupported" source_preload + +# Standard bare startup atomically owns new roots without Linux-specific tools. +printf '%s\n' 'backend=RoCkSdB' 'rocksdb.provider=rocksdb' \ + > "$SERVER_LAUNCHER_ROOT/selected graphs/hugegraph.properties" +printf '%s\n' '#!/bin/bash' 'echo unexpected-marker-initializer >&2; exit 99' \ + > "$SERVER_LAUNCHER_ROOT/bin/verify-rocksdb-provider.sh" +chmod +x "$SERVER_LAUNCHER_ROOT/bin/verify-rocksdb-provider.sh" +JAVA_HOME='' CLASSPATH='' JAVA_CAPTURE="$SERVER_INIT_CAPTURE" PATH="$FAKE_BIN:$PATH" \ + bash "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" >/dev/null 2>&1 +for storage in data wal; do + grep -qx provider=rocksdb "$SERVER_LAUNCHER_ROOT/rocksdb-data/$storage/.hugegraph-rocksdb-provider" || + fail "standard guard did not claim its new root" +done +echo "PASS: standard bare init claims new roots without Linux tools" +mkdir -p "$SERVER_LAUNCHER_ROOT/rocksdb-data" +printf '%s\n' format=1 component=server provider=topling \ + > "$SERVER_LAUNCHER_ROOT/rocksdb-data/.hugegraph-rocksdb-provider" +cp "$SERVER_LAUNCHER_ROOT/rocksdb-data/.hugegraph-rocksdb-provider" "$TEST_ROOT/original-marker" +expect_failure "standard bare init rejects a conflicting marker" "provider marker mismatch" \ + env JAVA_HOME='' PATH="$FAKE_BIN:$PATH" bash "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" +expect_failure "standard direct launcher rejects a conflicting marker" "provider marker mismatch" \ + env JAVA_HOME='' STDOUT_MODE=true PATH="$FAKE_BIN:$PATH" \ + bash "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/conf/gremlin-server.yaml" 'conf/selected rest.properties' true +cmp "$TEST_ROOT/original-marker" "$SERVER_LAUNCHER_ROOT/rocksdb-data/.hugegraph-rocksdb-provider" + +# Even unmarked standard paths cannot redirect through a configured symlink. +rm "$SERVER_LAUNCHER_ROOT/rocksdb-data/.hugegraph-rocksdb-provider" +mkdir -p "$SERVER_LAUNCHER_ROOT/unmarked-data" +rm "$SERVER_LAUNCHER_ROOT/rocksdb-data/data/.hugegraph-rocksdb-provider" +rmdir "$SERVER_LAUNCHER_ROOT/rocksdb-data/data" +ln -s "$SERVER_LAUNCHER_ROOT/unmarked-data" "$SERVER_LAUNCHER_ROOT/rocksdb-data/data" +expect_failure "standard direct launcher rejects an unmarked data symlink" "contains a symlink" \ + env JAVA_HOME='' STDOUT_MODE=true PATH="$FAKE_BIN:$PATH" \ + bash "$SERVER_LAUNCHER_ROOT/bin/hugegraph-server.sh" \ + "$SERVER_LAUNCHER_ROOT/conf/gremlin-server.yaml" 'conf/selected rest.properties' true +expect_failure "standard bare init rejects an unmarked data symlink" "contains a symlink" \ + env JAVA_HOME='' PATH="$FAKE_BIN:$PATH" bash "$SERVER_LAUNCHER_ROOT/bin/init-store.sh" +[ -L "$SERVER_LAUNCHER_ROOT/rocksdb-data/data" ] || fail "guard changed the configured symlink" +[ -z "$(ls -A "$SERVER_LAUNCHER_ROOT/unmarked-data")" ] || fail "guard changed the symlink destination" + +source "$DIST_ROOT/src/assembly/static/bin/rocksdb-server-config.sh" +expect_failure "standard pre-open rejects filesystem root" "data path cannot be /" \ + server_verify_storage_path "$SERVER_LAUNCHER_ROOT" rocksdb / false +expect_failure "Topling pre-open rejects filesystem root" "data path cannot be /" \ + server_verify_storage_path "$SERVER_LAUNCHER_ROOT" topling / false diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-server-entrypoint-roots.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-server-entrypoint-roots.sh new file mode 100755 index 0000000000..6b708c8b39 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-topling-server-entrypoint-roots.sh @@ -0,0 +1,261 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Real marker-helper filesystem checks; run on Linux (no JVM/native runtime). +set -Eeuo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DIST_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +TEST_ROOT=$(mktemp -d) +trap 'rm -rf "$TEST_ROOT"' EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } +reset_fixture() { + rm -rf "$TEST_ROOT/server" + FIXTURE="$TEST_ROOT/server" + mkdir -p "$FIXTURE/bin" "$FIXTURE/conf/graphs" "$FIXTURE/topling-data" \ + "$FIXTURE/extra root/data" "$FIXTURE/extra root/wal" + cp "${ENTRYPOINT_SOURCE:-$DIST_ROOT/docker/docker-entrypoint.sh}" "$FIXTURE/entrypoint.sh" + cp "$DIST_ROOT/src/assembly/static/bin/"{rocksdb-server-config,verify-rocksdb-provider}.sh "$FIXTURE/bin/" + printf '%s\n' '#!/bin/bash' 'touch init-reached' 'exit 37' > "$FIXTURE/bin/init-store.sh" + printf '%s\n' '#!/bin/bash' 'exit 0' > "$FIXTURE/bin/wait-storage.sh" + chmod +x "$FIXTURE/bin/"*.sh + printf '%s\n' 'graphs=./conf/graphs' > "$FIXTURE/conf/rest-server.properties" + printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=rocksdb' > "$FIXTURE/conf/graphs/hugegraph.properties" + printf '%s\n' 'backend=rocksdb' 'rocksdb.provider=topling' \ + 'rocksdb.data_path=./extra root/data' 'rocksdb.wal_path=./extra root/wal' \ + > "$FIXTURE/conf/graphs/extra.properties" +} +marker() { + printf '%s\n' format=1 component=server "provider=$2" > "$1/.hugegraph-rocksdb-provider" +} +run_entrypoint() { + local status=0 + (cd "$FIXTURE"; HG_SERVER_ROCKSDB_PROVIDER=topling HG_SERVER_ENFORCE_PROVIDER_MARKER=true \ + bash ./entrypoint.sh) > "$TEST_ROOT/output" 2>&1 || status=$? + return "$status" +} +expect_rejected() { + local name="$1" message="$2" + if run_entrypoint; then fail "$name unexpectedly succeeded"; fi + [ ! -e "$FIXTURE/init-reached" ] || { cat "$TEST_ROOT/output"; fail "$name reached init-store"; } + grep -Fq "$message" "$TEST_ROOT/output" || { cat "$TEST_ROOT/output"; fail "$name missing diagnostic"; } + echo "PASS: $name" +} +expect_init() { + local status=0 + run_entrypoint || status=$? + [ "$status" = 37 ] && [ -f "$FIXTURE/init-reached" ] || { + cat "$TEST_ROOT/output"; fail "valid configuration did not reach init-store"; + } +} + +for backend in ROCKSDB rocksdbsst; do + reset_fixture + export HG_SERVER_BACKEND="$backend" + expect_init + unset HG_SERVER_BACKEND + grep -qx "backend=$backend" "$FIXTURE/conf/graphs/hugegraph.properties" + grep -qx "rocksdb.data_path=$FIXTURE/topling-data/data" "$FIXTURE/conf/graphs/hugegraph.properties" + grep -qx "rocksdb.wal_path=$FIXTURE/topling-data/wal" "$FIXTURE/conf/graphs/hugegraph.properties" + grep -qx provider=topling "$FIXTURE/topling-data/.hugegraph-rocksdb-provider" + echo "PASS: Docker primary $backend generates and validates Topling roots" +done + +reset_fixture +printf '%s\n' 'backend=RoCkSdB' 'rocksdb.provider=topling' \ + 'rocksdb.data_path=./extra root/data' 'rocksdb.wal_path=./extra root/wal' \ + > "$FIXTURE/conf/graphs/extra.properties" +marker "$FIXTURE/extra root" rocksdb +expect_rejected "mixed-case backend roots cannot bypass markers" "provider marker mismatch" + +reset_fixture +marker "$FIXTURE/extra root" rocksdb +printf '%s\n' existing-data > "$FIXTURE/extra root/data/SENTINEL" +cp "$FIXTURE/extra root/.hugegraph-rocksdb-provider" "$TEST_ROOT/before-marker" +expect_rejected "extra graph wrong ancestor marker" "provider marker mismatch" +cmp "$TEST_ROOT/before-marker" "$FIXTURE/extra root/.hugegraph-rocksdb-provider" +grep -qx existing-data "$FIXTURE/extra root/data/SENTINEL" + +reset_fixture +marker "$FIXTURE/topling-data" rocksdb +cp "$FIXTURE/conf/graphs/hugegraph.properties" "$TEST_ROOT/original-graph" +expect_rejected "primary root conflict before config rewrite" "provider marker mismatch" +cmp "$TEST_ROOT/original-graph" "$FIXTURE/conf/graphs/hugegraph.properties" + +reset_fixture +marker "$FIXTURE/extra root/data" topling +marker "$FIXTURE/extra root/wal" rocksdb +expect_rejected "separate WAL provider mismatch" "provider marker mismatch" + +reset_fixture +marker "$FIXTURE/extra root" topling +mkdir -p "$FIXTURE/foreign" +marker "$FIXTURE/foreign" rocksdb +rmdir "$FIXTURE/extra root/data" +ln -s "$FIXTURE/foreign" "$FIXTURE/extra root/data" +expect_rejected "marked ancestor cannot hide data symlink" "contains a symlink" +grep -qx provider=rocksdb "$FIXTURE/foreign/.hugegraph-rocksdb-provider" + +reset_fixture +printf '%s\n' 'backend=rocksdb' > "$FIXTURE/conf/graphs/implicit.properties" +expect_rejected "mixed explicit and default providers" "conflicting rocksdb.provider values" + +reset_fixture +printf '%s\n' 'rocksdb.data_disks=g/vertex:/disk' >> "$FIXTURE/conf/graphs/extra.properties" +expect_rejected "unsupported additional disks" "does not support rocksdb.data_disks" + +reset_fixture +printf '%s\n' 'include=hidden.properties' >> "$FIXTURE/conf/graphs/extra.properties" +expect_rejected "unsupported config includes" "property includes are unsupported" + +for graph_path in './custom graphs' absolute; do + reset_fixture + mv "$FIXTURE/conf/graphs" "$FIXTURE/custom graphs" + if [ "$graph_path" = absolute ]; then graph_path="$FIXTURE/custom graphs"; fi + printf 'graphs=%s\n' "$graph_path" > "$FIXTURE/conf/rest-server.properties" + expect_init + grep -qx provider=topling "$FIXTURE/extra root/data/.hugegraph-rocksdb-provider" + grep -qx provider=topling "$FIXTURE/extra root/wal/.hugegraph-rocksdb-provider" + grep -qx rocksdb.provider=topling "$FIXTURE/custom graphs/hugegraph.properties" + echo "PASS: custom graphs $graph_path validates data and WAL before init" +done + +reset_fixture +marker "$FIXTURE/extra root" topling +expect_init +echo "PASS: matching deployment ancestor accepts configured data/WAL children" + +# A single DB bind mount must fail before validation claims any other graph +# path. The real mount is exercised separately in a Linux container. +reset_fixture +marker "$FIXTURE/topling-data" topling +mkdir -p "$FIXTURE/topling-data/data/g" "$TEST_ROOT/fake-bin" +cat > "$TEST_ROOT/fake-bin/mountpoint" <<'SH' +#!/bin/bash +[ "$1" = --version ] && { echo 'mountpoint from util-linux 2.41.3'; exit 0; } +[ "$1" = -q ] && [ "$2" = -- ] || exit 1 +[ "$3" = "$MOCK_MOUNTPOINT" ] && exit 0 +exit 32 +SH +chmod +x "$TEST_ROOT/fake-bin/mountpoint" +export MOCK_MOUNTPOINT="$FIXTURE/topling-data/data/g" +TEST_SAVED_PATH="$PATH" +export PATH="$TEST_ROOT/fake-bin:$PATH" +expect_rejected "single DB mount before graph claims" \ + "RocksDB directory cannot itself be a mount point" +[ ! -e "$FIXTURE/extra root/data/.hugegraph-rocksdb-provider" ] || + fail "mount rejection claimed another graph data path" +[ ! -e "$FIXTURE/topling-data/data/.hugegraph-rocksdb-provider" ] || + fail "mount rejection claimed the primary data path" + +reset_fixture +mkdir -p "$FIXTURE/extra root/data/g" +export MOCK_MOUNTPOINT="$FIXTURE/extra root/data/g" +expect_rejected "extra graph DB mount before primary root claim" \ + "RocksDB directory cannot itself be a mount point" +[ ! -e "$FIXTURE/topling-data/.hugegraph-rocksdb-provider" ] || + fail "mount rejection claimed the primary root" +[ ! -e "$FIXTURE/topling-data/.hugegraph-state" ] || + fail "mount rejection created primary state" +cat > "$TEST_ROOT/fake-bin/mountpoint" <<'SH' +#!/bin/bash +[ "$1" = --version ] && { echo 'BusyBox v1.37.0'; exit 0; } +exit 32 +SH +reset_fixture +expect_rejected "non util-linux mountpoint before root claim" \ + "util-linux mountpoint is required" + [ ! -e "$FIXTURE/topling-data/.hugegraph-rocksdb-provider" ] || + fail "non util-linux mountpoint claimed the primary root" +cat > "$TEST_ROOT/fake-bin/mountpoint" <<'SH' +#!/bin/bash +[ "$1" = --version ] && { echo 'mountpoint from util-linux 2.36.1'; exit 0; } +exit 1 +SH +reset_fixture +expect_rejected "old util-linux mountpoint before root claim" \ + "util-linux mountpoint 2.37+ is required" +export PATH="$TEST_SAVED_PATH" +unset MOCK_MOUNTPOINT + +# Direct launch and init-store run the real marker verifier before Java/native. +prepare_direct() { + reset_fixture + cp "$DIST_ROOT/src/assembly/static/bin/"{preload-topling,hugegraph-server,init-store,util}.sh "$FIXTURE/bin/" + mkdir -p "$FIXTURE/lib" "$FIXTURE/plugins" "$FIXTURE/ext" "$FIXTURE/logs" "$FIXTURE/fake-bin" + # shellcheck disable=SC2016 + printf '%s\n' '#!/bin/bash' 'touch "$JAVA_REACHED"' 'exit 98' > "$FIXTURE/fake-bin/java" + # shellcheck disable=SC2016 + printf '%s\n' '#!/bin/bash' 'case "$1" in -s) echo Linux;; -m) echo x86_64;; *) echo Linux;; esac' \ + > "$FIXTURE/fake-bin/uname" + chmod +x "$FIXTURE/fake-bin/"* "$FIXTURE/bin/"*.sh + # Custom graph directory and mixed-case backend must reach the same guard. + mv "$FIXTURE/conf/graphs" "$FIXTURE/custom graphs" + rm "$FIXTURE/custom graphs/extra.properties" + printf '%s\n' 'graphs=./custom graphs' > "$FIXTURE/conf/rest-server.properties" + printf '%s\n' 'backend=RoCkSdB' 'rocksdb.provider=topling' \ + 'rocksdb.data_path=./extra root/data' 'rocksdb.wal_path=./extra root/wal' \ + > "$FIXTURE/custom graphs/hugegraph.properties" +} +expect_direct_rejected() { + local launcher="$1" name="$2" message="$3" status=0 + local args=() + if [ "$launcher" = hugegraph-server ]; then + args=(conf/gremlin-server.yaml conf/rest-server.properties true) + fi + (cd "$FIXTURE"; JAVA_HOME='' STDOUT_MODE=true JAVA_REACHED="$FIXTURE/java-reached" \ + PATH="$FIXTURE/fake-bin:$PATH" bash "./bin/$launcher.sh" "${args[@]}") \ + > "$TEST_ROOT/output" 2>&1 || status=$? + [ "$status" != 0 ] || fail "$name unexpectedly succeeded" + [ ! -e "$FIXTURE/java-reached" ] || { cat "$TEST_ROOT/output"; fail "$name reached Java"; } + grep -Fq "$message" "$TEST_ROOT/output" || { cat "$TEST_ROOT/output"; fail "$name missing diagnostic"; } + echo "PASS: $name" +} +for launcher in hugegraph-server init-store; do + prepare_direct + marker "$FIXTURE/extra root" rocksdb + printf '%s\n' original-data > "$FIXTURE/extra root/data/SENTINEL" + cp "$FIXTURE/extra root/.hugegraph-rocksdb-provider" "$TEST_ROOT/original-marker" + expect_direct_rejected "$launcher" "$launcher rejects conflicting ancestor marker" "provider marker mismatch" + cmp "$TEST_ROOT/original-marker" "$FIXTURE/extra root/.hugegraph-rocksdb-provider" + grep -qx original-data "$FIXTURE/extra root/data/SENTINEL" + + prepare_direct + printf '%s\n' original-data > "$FIXTURE/extra root/data/SENTINEL" + expect_direct_rejected "$launcher" "$launcher rejects unmarked nonempty Topling root" \ + "refusing unmarked non-empty data path" + grep -qx original-data "$FIXTURE/extra root/data/SENTINEL" + [ ! -e "$FIXTURE/extra root/data/.hugegraph-rocksdb-provider" ] || fail "rejected data was marked" + + prepare_direct + mkdir -p "$FIXTURE/extra root/data/g" + cat > "$FIXTURE/fake-bin/mountpoint" <<'SH' +#!/bin/bash +[ "$1" = --version ] && { echo 'mountpoint from util-linux 2.41.3'; exit 0; } +[ "$1" = -q ] && [ "$2" = -- ] || exit 1 +[ "$3" = "$MOCK_MOUNTPOINT" ] && exit 0 +exit 32 +SH + chmod +x "$FIXTURE/fake-bin/mountpoint" + export MOCK_MOUNTPOINT="$FIXTURE/extra root/data/g" + expect_direct_rejected "$launcher" "$launcher rejects mounted DB before Java" \ + "RocksDB directory cannot itself be a mount point" + unset MOCK_MOUNTPOINT + [ ! -e "$FIXTURE/extra root/data/.hugegraph-rocksdb-provider" ] || + fail "mount rejection claimed direct launcher data path" +done diff --git a/hugegraph-server/hugegraph-rocksdb/pom.xml b/hugegraph-server/hugegraph-rocksdb/pom.xml index 845cf40f9c..276f8163cd 100644 --- a/hugegraph-server/hugegraph-rocksdb/pom.xml +++ b/hugegraph-server/hugegraph-rocksdb/pom.xml @@ -37,7 +37,7 @@ org.rocksdb rocksdbjni - 8.10.2 + ${rocksdb.version} diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java index 2aa6d2292c..20142ade02 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/OpenedRocksDB.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; @@ -44,9 +45,16 @@ public class OpenedRocksDB implements AutoCloseable { private final RocksDB rocksdb; private final Map cfHandles; private final SstFileManager sstFileManager; + private final FileChannel recoveryLock; public OpenedRocksDB(RocksDB rocksdb, Map cfHandles, SstFileManager sstFileManager) { + this(rocksdb, cfHandles, sstFileManager, null); + } + + OpenedRocksDB(RocksDB rocksdb, Map cfHandles, + SstFileManager sstFileManager, FileChannel recoveryLock) { + this.recoveryLock = recoveryLock; this.rocksdb = rocksdb; this.cfHandles = cfHandles; this.sstFileManager = sstFileManager; @@ -82,6 +90,29 @@ public boolean isOwningHandle() { @Override public void close() { + try { + this.closeNative(); + } finally { + if (!this.isOwningHandle()) { + RocksDBSnapshotRestore.unlock(this.recoveryLock); + } + } + } + + // Transfer the existing lock to the replacement DB without an unlocked gap. + FileChannel closeForRestore() { + try { + this.closeNative(); + } catch (RuntimeException | Error e) { + if (!this.isOwningHandle()) { + RocksDBSnapshotRestore.unlock(this.recoveryLock); + } + throw e; + } + return this.recoveryLock != null && this.recoveryLock.isOpen() ? this.recoveryLock : null; + } + + private void closeNative() { if (!this.isOwningHandle()) { return; } @@ -89,7 +120,6 @@ public void close() { cf.close(); } this.cfHandles.clear(); - this.rocksdb.close(); } diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java index 17148e0944..0f0d18f1a3 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBOptions.java @@ -61,6 +61,14 @@ public static synchronized RocksDBOptions instance() { "rocksdb-data/data" ); + public static final ConfigOption PROVIDER = + new ConfigOption<>( + "rocksdb.provider", + "The RocksDB-compatible runtime provider.", + allowValues("rocksdb", "topling"), + "rocksdb" + ); + public static final ConfigListOption DATA_DISKS = new ConfigListOption<>( "rocksdb.data_disks", diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java index 7d0576e369..c5f3867ef6 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSessions.java @@ -17,6 +17,9 @@ package org.apache.hugegraph.backend.store.rocksdb; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Set; @@ -59,6 +62,40 @@ public RocksDBSessions(HugeConfig config, String database, String store) { public abstract void forceCloseRocksDB(); + public boolean databaseOpened() { + return this.opened(); + } + + public void clearTables(Collection tables) { + Session session = (Session) this.useSession(); + try { + List>> ranges = + new ArrayList<>(); + for (String table : tables) { + Pair range = session.keyRange(table); + if (range != null) { + ranges.add(Pair.of(table, range)); + } + } + + for (Pair> entry : ranges) { + String table = entry.getLeft(); + byte[] first = entry.getRight().getLeft(); + byte[] last = entry.getRight().getRight(); + if (!Arrays.equals(first, last)) { + session.deleteRange(table, first, last); + } + session.delete(table, last); + } + session.commit(); + } catch (Throwable e) { + session.rollback(); + throw e; + } finally { + this.close(); + } + } + @Override public abstract Session session(); diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSnapshotRestore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSnapshotRestore.java new file mode 100644 index 0000000000..9f06454171 --- /dev/null +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSnapshotRestore.java @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.backend.store.rocksdb; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.backend.BackendException; +import org.apache.hugegraph.util.Log; +import org.slf4j.Logger; + +/** + * Fail-closed snapshot installation. The marker lives outside the data directory + * and the original checkpoint remains the recovery source until after reopening. + * A subsequent open retries the recorded checkpoint before native recovery runs. + */ +final class RocksDBSnapshotRestore { + + private static final Logger LOG = Log.logger(RocksDBSnapshotRestore.class); + + private final File data; + private final File wal; + private final File snapshot; + private final Path marker; + private final FileOperations files; + private final Path configuredWal; + private final List walLinks = new ArrayList<>(); + private final List walLinkTargets = new ArrayList<>(); + + RocksDBSnapshotRestore(String data, String wal, String snapshot, + FileOperations files) throws IOException { + this.data = new File(data).getCanonicalFile(); + this.wal = wal == null || wal.isEmpty() ? this.data : + new File(wal).getCanonicalFile(); + this.snapshot = new File(snapshot).getCanonicalFile(); + this.marker = marker(this.data.toString()); + this.files = files; + this.configuredWal = wal == null || wal.isEmpty() ? this.data.toPath() : + new File(wal).toPath().toAbsolutePath().normalize(); + Path component = this.configuredWal.getRoot(); + for (Path name : this.configuredWal) { + component = component.resolve(name); + if (Files.isSymbolicLink(component)) { + this.walLinks.add(component); + this.walLinkTargets.add(Files.readSymbolicLink(component)); + } + } + if (overlaps(this.snapshot, this.data) || + this.wal.toPath().startsWith(this.snapshot.toPath())) { + throw new IOException("Checkpoint must not overlap data or WAL: " + this.snapshot); + } + validateCheckpoint(this.snapshot); + } + + private static void validateCheckpoint(File snapshot) throws IOException { + Path current = new File(snapshot, "CURRENT").toPath(); + if (!Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS) || Files.size(current) > 128L) { + throw new IOException("Checkpoint CURRENT is missing or invalid: " + snapshot); + } + String manifest = new String(Files.readAllBytes(current), StandardCharsets.UTF_8).trim(); + if (!manifest.matches("MANIFEST-[0-9]+") || + !Files.isRegularFile(new File(snapshot, manifest).toPath(), LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Checkpoint MANIFEST is missing or invalid: " + snapshot); + } + } + + private static boolean overlaps(File first, File second) { + return first.toPath().startsWith(second.toPath()) || + second.toPath().startsWith(first.toPath()); + } + + private static Path marker(String data) throws IOException { + return new File(new File(data).getCanonicalPath() + ".resume-pending").toPath(); + } + + static FileChannel lock(String data) { + FileChannel channel = null; + try { + Path directory = new File(data).getCanonicalFile().toPath(); + Path parent = directory.getParent(); + if (parent == null) { + throw new IOException("Database directory cannot be a filesystem root: " + directory); + } + // Sibling guards must share the DB's backing parent. Binding just the + // DB directory hides those guards at aliases and cannot be replaced + // during restore (EBUSY). Mount the parent data root instead. + if (Files.isDirectory(directory) && + (System.getProperty("os.name").startsWith("Linux") ? isLinuxMountPoint(directory) : + !Files.getFileStore(directory).equals(Files.getFileStore(parent)))) { + throw new IOException("Database directory cannot itself be a mount point; " + + "mount its parent data root instead: " + directory); + } + Path path = new File(directory + ".resume-lock").toPath(); + Files.createDirectories(path.getParent()); + channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + if (channel.tryLock() == null) { + throw new IOException("Database is already open or recovering: " + data); + } + return channel; + } catch (IOException | OverlappingFileLockException e) { + unlock(channel); + throw new BackendException("Cannot lock database for open/recovery: '%s'", e, data); + } + } + + private static boolean isLinuxMountPoint(Path directory) throws IOException { + // FileStore equality cannot detect a bind mount within the same filesystem. + // Read this process's namespace; kernel mountinfo escapes whitespace and backslashes. + for (String mount : Files.readAllLines(Paths.get("/proc/self/mountinfo"), + StandardCharsets.UTF_8)) { + String[] fields = mount.split(" ", 7); + if (fields.length < 7) { + throw new IOException("Invalid Linux mountinfo record"); + } + String path = fields[4].replace("\\040", " ").replace("\\011", "\t") + .replace("\\012", "\n").replace("\\134", "\\"); + if (directory.equals(Paths.get(path))) { + return true; + } + } + return false; + } + + static void unlock(FileChannel channel) { + if (channel != null) { + try { + channel.close(); + } catch (IOException e) { + throw new BackendException("Failed to release database recovery lock", e); + } + } + } + + static void start(String data, String wal, String snapshot) throws IOException { + Path marker = marker(data); + if (Files.exists(marker, LinkOption.NOFOLLOW_LINKS)) { + Properties pending = new Properties(); + try (InputStream input = Files.newInputStream(marker)) { + pending.load(input); + } + Path configuredWal = wal == null || wal.isEmpty() ? new File(data).getCanonicalFile().toPath() : + new File(wal).toPath().toAbsolutePath().normalize(); + if (!new File(snapshot).getCanonicalPath().equals(pending.getProperty("snapshot")) || + !configuredWal.toString().equals(pending.getProperty("configured-wal"))) { + throw new IOException("Another or incomplete checkpoint restore is pending: " + marker); + } + // prepareOpen will reconstruct links and revalidate the recorded source. + return; + } + new RocksDBSnapshotRestore(data, wal, snapshot, new FileOperations()).begin(); + } + + void begin() throws IOException { + Properties state = new Properties(); + state.setProperty("snapshot", this.snapshot.toString()); + state.setProperty("wal", this.wal.toString()); + state.setProperty("configured-wal", this.configuredWal.toString()); + state.setProperty("wal-links", Integer.toString(this.walLinks.size())); + for (int i = 0; i < this.walLinks.size(); i++) { + state.setProperty("wal-link-" + i, this.walLinks.get(i).toString()); + state.setProperty("wal-link-target-" + i, this.walLinkTargets.get(i).toString()); + } + // CREATE_NEW prevents replacing the only description of an interrupted restore. + try (OutputStream output = Files.newOutputStream(this.marker, + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE)) { + state.store(output, "Pending RocksDB checkpoint restore; do not remove"); + } catch (FileAlreadyExistsException e) { + Properties pending = new Properties(); + try (InputStream input = Files.newInputStream(this.marker)) { + pending.load(input); + } + if (!state.equals(pending)) { + throw new IOException("Another checkpoint restore is pending: " + this.marker, e); + } + } + try (FileChannel channel = FileChannel.open(this.marker, StandardOpenOption.WRITE)) { + channel.force(true); + } + } + + static RocksDBSnapshotRestore prepareOpen(String data, String wal) { + try { + Path marker = marker(data); + if (!Files.exists(marker, LinkOption.NOFOLLOW_LINKS)) { + return null; + } + if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Invalid recovery marker " + marker); + } + Properties state = new Properties(); + try (InputStream input = Files.newInputStream(marker)) { + state.load(input); + } + String snapshot = state.getProperty("snapshot"); + String savedWal = state.getProperty("wal"); + if (snapshot == null || savedWal == null) { + throw new IOException("Incomplete recovery marker " + marker); + } + Path configuredWal = wal == null || wal.isEmpty() ? new File(data).getCanonicalFile().toPath() : + new File(wal).toPath().toAbsolutePath().normalize(); + if (!configuredWal.toString().equals(state.getProperty("configured-wal"))) { + throw new IOException("WAL configuration changed: " + marker); + } + int links; + try { + links = Integer.parseInt(state.getProperty("wal-links")); + } catch (NumberFormatException e) { + throw new IOException("Invalid WAL links in recovery marker " + marker, e); + } + if (links < 0 || links > configuredWal.getNameCount()) { + throw new IOException("Invalid WAL link count: " + marker); + } + for (int i = 0; i < links; i++) { + String link = state.getProperty("wal-link-" + i); + String target = state.getProperty("wal-link-target-" + i); + if (link == null || target == null || !configuredWal.startsWith(new File(link).toPath())) { + throw new IOException("Invalid WAL link in recovery marker " + marker); + } + restoreLink(new File(link).toPath(), new File(target).toPath()); + } + RocksDBSnapshotRestore restore = + new RocksDBSnapshotRestore(data, wal, snapshot, new FileOperations()); + if (!restore.wal.toString().equals(savedWal)) { + throw new IOException("WAL configuration changed during recovery: " + marker); + } + restore.install(); + return restore; + } catch (IOException e) { + throw new BackendException("Pending snapshot recovery failed for '%s'; " + + "preserve the .resume-pending marker and checkpoint", e, data); + } + } + + void install() throws IOException { + // Copy, never move: partial data copies and WAL failures must be retryable. + this.files.deleteDirectory(this.data); + this.files.copyDirectory(this.snapshot, this.data); + verifyTree(this.snapshot.toPath(), this.data.toPath()); + for (int i = 0; i < this.walLinks.size(); i++) { + restoreLink(this.walLinks.get(i), this.walLinkTargets.get(i)); + } + installWal(this.data, this.wal, this.files); + } + + private static void verifyTree(Path source, Path copy) throws IOException { + try (Stream paths = Files.walk(source)) { + for (Path path : (Iterable) paths::iterator) { + Path target = copy.resolve(source.relativize(path)); + if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + if (!Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Missing checkpoint directory: " + target); + } + } else if (Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS) && + Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + verify(path.toFile(), target.toFile()); + } else { + throw new IOException("Invalid checkpoint file: " + path); + } + } + } + } + + private static void restoreLink(Path link, Path target) throws IOException { + if (!Files.exists(link, LinkOption.NOFOLLOW_LINKS)) { + Files.createDirectories(link.getParent()); + Files.createSymbolicLink(link, target); + } else if (!Files.isSymbolicLink(link) || !Files.readSymbolicLink(link).equals(target)) { + throw new IOException("WAL link changed during recovery: " + link); + } + } + + void complete() { + try { + Files.delete(this.marker); + } catch (IOException e) { + throw new BackendException("Failed to finish snapshot recovery at '%s'", e, this.marker); + } + // Restore historically consumes its source. Only clean it after native + // reopen succeeded AND the marker no longer directs retries to it. + try { + FileUtils.deleteDirectory(this.snapshot); + } catch (IOException e) { + LOG.warn("Snapshot restored but source cleanup failed: {}", this.snapshot, e); + } + } + + static void installWal(File data, File wal, FileOperations files) throws IOException { + data = data.getCanonicalFile(); + wal = wal.getCanonicalFile(); + if (data.equals(wal)) { + return; + } + List source = logs(data); + FileUtils.forceMkdir(wal); + Path staging = Files.createTempDirectory(wal.toPath(), ".resume-staging-"); + try { + for (File log : source) { + File staged = staging.resolve(log.getName()).toFile(); + files.copyFile(log, staged); + verify(log, staged); + } + // Retire old logs before publishing any checkpoint log. Directory moves + // would move nested data or replace the user's WAL symlink. + for (File old : logs(wal)) { + files.move(old.toPath(), staging.resolve(old.getName() + ".aside")); + } + for (File log : source) { + files.move(staging.resolve(log.getName()), new File(wal, log.getName()).toPath()); + } + List installed = logs(wal); + if (installed.size() != source.size()) { + throw new IOException("Unexpected WAL files during checkpoint restore: " + wal); + } + for (File log : source) { + verify(log, new File(wal, log.getName())); + } + for (File log : source) { + Files.delete(log.toPath()); + } + } finally { + // Source checkpoint is untouched, including when cleanup itself fails. + FileUtils.deleteDirectory(staging.toFile()); + } + } + + private static void verify(File source, File copy) throws IOException { + if (!FileUtils.contentEquals(source, copy)) { + throw new IOException("Checkpoint WAL content mismatch: " + copy); + } + } + + private static List logs(File directory) throws IOException { + File[] children = directory.listFiles(); + if (children == null) { + throw new IOException("Cannot list WAL directory " + directory); + } + List logs = new ArrayList<>(); + for (File child : children) { + if (child.getName().matches("[0-9]+\\.log")) { + if (!Files.isRegularFile(child.toPath(), LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("WAL must be a regular file: " + child); + } + logs.add(child); + } + } + return logs; + } + + // Package-private seam for deterministic IO faults, without mocking native DBs. + static class FileOperations { + + void copyDirectory(File source, File target) throws IOException { + FileUtils.copyDirectory(source, target); + } + + void deleteDirectory(File directory) throws IOException { + FileUtils.deleteDirectory(directory); + } + + void copyFile(File source, File target) throws IOException { + FileUtils.copyFile(source, target); + } + + void move(Path source, Path target) throws IOException { + Files.move(source, target); + } + } +} diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java index c1cc1c5075..5dd5d58f80 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStdSessions.java @@ -18,6 +18,9 @@ package org.apache.hugegraph.backend.store.rocksdb; import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.OverlappingFileLockException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -64,6 +67,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.SstFileManager; +import org.rocksdb.Status; import org.rocksdb.TableFormatConfig; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; @@ -83,23 +87,28 @@ public class RocksDBStdSessions extends RocksDBSessions { private final AtomicInteger refCount; public RocksDBStdSessions(HugeConfig config, String database, String store, - String dataPath, String walPath) throws RocksDBException { + String dataPath, String walPath) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; + this.rocksdb = RocksDBStdSessions.openRocksDB(config, dataPath, walPath); this.refCount = new AtomicInteger(1); } public RocksDBStdSessions(HugeConfig config, String database, String store, String dataPath, String walPath, - List cfNames) throws RocksDBException { + List cfNames) throws + RocksDBException { super(config, database, store); this.config = config; this.dataPath = dataPath; this.walPath = walPath; - this.rocksdb = RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath); + + this.rocksdb = + RocksDBStdSessions.openRocksDB(config, cfNames, dataPath, walPath); this.refCount = new AtomicInteger(1); this.ingestExternalFile(); @@ -202,17 +211,15 @@ public boolean existsTable(String table) { } @Override - public void reloadRocksDB() throws RocksDBException { - if (this.rocksdb.isOwningHandle()) { - this.rocksdb.close(); - } + public synchronized void reloadRocksDB() throws RocksDBException { + FileChannel recoveryLock = this.rocksdb.closeForRestore(); this.rocksdb = RocksDBStdSessions.openRocksDB(this.config, ImmutableList.of(), - this.dataPath, this.walPath); + this.dataPath, this.walPath, recoveryLock); } @Override - public void forceCloseRocksDB() { - this.rocksdb().close(); + public synchronized void forceCloseRocksDB() { + this.rocksdb.close(); } @Override @@ -256,30 +263,28 @@ public void createSnapshot(String snapshotPath) { } @Override - public void resumeSnapshot(String snapshotPath) { - File originDataDir = new File(this.dataPath); + public synchronized void resumeSnapshot(String snapshotPath) { File snapshotDir = new File(snapshotPath); + FileChannel recoveryLock = null; + boolean transferred = false; try { - /* - * Close current instance first - * NOTE: must close rocksdb instance before deleting file directory, - * if close after copying the snapshot directory to origin position, - * it may produce dirty data. - */ - this.forceCloseRocksDB(); - // Delete origin data directory - if (originDataDir.exists()) { - LOG.info("Delete origin data directory {}", originDataDir); - FileUtils.deleteDirectory(originDataDir); + // Close the native handle before replacing files, but retain the + // sibling lock throughout marker creation, installation and reopen. + recoveryLock = this.rocksdb.closeForRestore(); + if (recoveryLock == null) { + recoveryLock = lockForOpen(this.dataPath); } - // Move snapshot directory to origin data directory - FileUtils.moveDirectory(snapshotDir, originDataDir); - LOG.info("Move snapshot directory {} to {}", snapshotDir, originDataDir); - // Reload rocksdb instance - this.reloadRocksDB(); + RocksDBSnapshotRestore.start(this.dataPath, this.walPath, snapshotPath); + this.rocksdb = RocksDBStdSessions.openRocksDB(this.config, ImmutableList.of(), + this.dataPath, this.walPath, recoveryLock); + transferred = true; } catch (Exception e) { - throw new BackendException("Failed to resume snapshot '%s' to' %s'", + throw new BackendException("Failed to resume snapshot '%s' to '%s'", e, snapshotDir, this.dataPath); + } finally { + if (!transferred) { + RocksDBSnapshotRestore.unlock(recoveryLock); + } } } @@ -334,6 +339,13 @@ private void checkValid() { E.checkState(this.rocksdb.isOwningHandle(), "It seems RocksDB has been closed"); } + private void replaceSeparateWalDirectory() throws IOException { + if (this.walPath != null && !this.walPath.isEmpty()) { + RocksDBSnapshotRestore.installWal(new File(this.dataPath), new File(this.walPath), + new RocksDBSnapshotRestore.FileOperations()); + } + } + private RocksDB rocksdb() { this.checkValid(); return this.rocksdb.rocksdb(); @@ -365,61 +377,139 @@ private void ingestExternalFile() throws RocksDBException { } } + private static void finishRestore(RocksDBSnapshotRestore restore, OpenedRocksDB opened) { + if (restore != null) { + try { + restore.complete(); + } catch (RuntimeException e) { + opened.close(); + throw e; + } + } + } + private static OpenedRocksDB openRocksDB(HugeConfig config, String dataPath, - String walPath) throws RocksDBException { - // Init options - Options options = new Options(); - RocksDBStdSessions.initOptions(config, options, options, options, options); - options.setWalDir(walPath); - SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); - options.setSstFileManager(sstFileManager); - /* - * Open RocksDB at the first time - * Don't merge old CFs, we expect a clear DB when using this one - */ - RocksDB rocksdb = RocksDB.open(options, dataPath); - Map cfs = new ConcurrentHashMap<>(); - return new OpenedRocksDB(rocksdb, cfs, sstFileManager); + String walPath) throws + RocksDBException { + FileChannel recoveryLock = lockForOpen(dataPath); + try { + RocksDBSnapshotRestore restore = RocksDBSnapshotRestore.prepareOpen(dataPath, walPath); + // Init options + Options options = new Options(); + RocksDBStdSessions.initOptions(config, options, options, options, options); + options.setWalDir(walPath); + SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); + options.setSstFileManager(sstFileManager); + + RocksDB rocksdb = RocksDB.open(options, dataPath); + + Map cfs = new ConcurrentHashMap<>(); + OpenedRocksDB opened = new OpenedRocksDB(rocksdb, cfs, sstFileManager, recoveryLock); + finishRestore(restore, opened); + return opened; + } catch (RuntimeException | RocksDBException | Error e) { + RocksDBSnapshotRestore.unlock(recoveryLock); + throw e; + } } private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, String dataPath, - String walPath) throws RocksDBException { - // Old CFs should always be opened - Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, - cfNames); - List cfs = ImmutableList.copyOf(mergedCFs); - - // Init CFs options - List cfds = new ArrayList<>(cfs.size()); - for (String cf : cfs) { - ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(cf)); - ColumnFamilyOptions options = cfd.getOptions(); - RocksDBStdSessions.initOptions(config, null, null, options, options); - cfds.add(cfd); + String walPath) throws + RocksDBException { + return openRocksDB(config, cfNames, dataPath, walPath, null); + } + + private static OpenedRocksDB openRocksDB(HugeConfig config, List cfNames, + String dataPath, String walPath, + FileChannel heldLock) throws RocksDBException { + FileChannel recoveryLock = heldLock != null ? heldLock : lockForOpen(dataPath); + try { + RocksDBSnapshotRestore restore = RocksDBSnapshotRestore.prepareOpen(dataPath, walPath); + // Old CFs should always be opened + Set mergedCFs = RocksDBStdSessions.mergeOldCFs(dataPath, + cfNames); + List cfs = ImmutableList.copyOf(mergedCFs); + + // Init CFs options + List cfds = new ArrayList<>(cfs.size()); + for (String cf : cfs) { + ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(cf)); + ColumnFamilyOptions options = cfd.getOptions(); + RocksDBStdSessions.initOptions(config, null, null, options, options); + cfds.add(cfd); + } + + // Init DB options + DBOptions options = new DBOptions(); + RocksDBStdSessions.initOptions(config, options, options, null, null); + if (walPath != null) { + options.setWalDir(walPath); + } + SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); + options.setSstFileManager(sstFileManager); + // Open RocksDB with CFs + List cfhs = new ArrayList<>(); + + RocksDB rocksdb = RocksDB.open(options, dataPath, cfds, cfhs); + + E.checkState(cfhs.size() == cfs.size(), + "Expect same size of cf-handles and cf-names"); + // Collect CF Handles + Map cfHandles = new ConcurrentHashMap<>(); + for (int i = 0; i < cfs.size(); i++) { + cfHandles.put(cfs.get(i), new OpenedRocksDB.CFHandle(rocksdb, cfhs.get(i))); + } + OpenedRocksDB opened = new OpenedRocksDB(rocksdb, cfHandles, sstFileManager, recoveryLock); + finishRestore(restore, opened); + return opened; + } catch (RuntimeException | RocksDBException | Error e) { + RocksDBSnapshotRestore.unlock(recoveryLock); + throw e; } + } - // Init DB options - DBOptions options = new DBOptions(); - RocksDBStdSessions.initOptions(config, options, options, null, null); - if (walPath != null) { - options.setWalDir(walPath); + private static FileChannel lockForOpen(String dataPath) throws RocksDBException { + try { + return RocksDBSnapshotRestore.lock(dataPath); + } catch (BackendException e) { + // Opening a session must retain the checked RocksDB failure contract, + // including the contention classification used by Store's shared-CF path. + Throwable cause = e.getCause(); + String message; + boolean contention = false; + if (cause instanceof OverlappingFileLockException) { + contention = true; + message = "IO error: lock hold by current process: " + + dataPath + ".resume-lock: No locks available " + + "(database open/recovery lock)"; + } else if (cause instanceof IOException && cause.getMessage() != null && + cause.getMessage().startsWith("Database is already open or recovering:")) { + contention = true; + message = "IO error: database open/recovery lock held: " + + dataPath + ": No locks available"; + } else { + String reason = cause == null ? "" : ": " + cause; + message = e.getMessage() + reason; + } + throw new RecoveryLockException(message, contention); } - SstFileManager sstFileManager = new SstFileManager(Env.getDefault()); - options.setSstFileManager(sstFileManager); + } - // Open RocksDB with CFs - List cfhs = new ArrayList<>(); - RocksDB rocksdb = RocksDB.open(options, dataPath, cfds, cfhs); + static final class RecoveryLockException extends RocksDBException { + + private static final long serialVersionUID = 1L; + + private final boolean contention; + + RecoveryLockException(String message, boolean contention) { + super(message, new Status(Status.Code.IOError, Status.SubCode.None, message)); + this.contention = contention; + } - E.checkState(cfhs.size() == cfs.size(), - "Expect same size of cf-handles and cf-names"); - // Collect CF Handles - Map cfHandles = new ConcurrentHashMap<>(); - for (int i = 0; i < cfs.size(); i++) { - cfHandles.put(cfs.get(i), new OpenedRocksDB.CFHandle(rocksdb, cfhs.get(i))); + boolean isContention() { + return this.contention; } - return new OpenedRocksDB(rocksdb, cfHandles, sstFileManager); } private static Set mergeOldCFs(String path, diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java index 3b6b54eadb..f2c419e8a1 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -103,6 +103,7 @@ public abstract class RocksDBStore extends AbstractBackendStore(); this.dbs = new ConcurrentHashMap<>(); this.storeLock = new ReentrantReadWriteLock(); + this.toplingProvider = false; this.registerMetaHandlers(); } @@ -212,6 +214,8 @@ public synchronized void open(HugeConfig config) { E.checkNotNull(config, "config"); String graphStore = config.get(CoreOptions.STORE_GRAPH); this.isGraphStore = this.store.equals(graphStore); + this.toplingProvider = "topling".equals( + config.get(RocksDBOptions.PROVIDER)); this.dataPath = config.get(RocksDBOptions.DATA_PATH); if (this.sessions != null && !this.sessions.closed()) { @@ -313,8 +317,10 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, walPath, tableNames); } catch (RocksDBException e) { RocksDBSessions origin = this.dbs.get(dataPath); - if (origin != null) { - if (e.getMessage().contains("No locks available")) { + if (origin != null && + e instanceof RocksDBStdSessions.RecoveryLockException && + ((RocksDBStdSessions.RecoveryLockException) e).isContention()) { + synchronized (origin) { /* * Open twice, copy a RocksDBSessions reference, since from * v0.11.2 release we don't support multi graphs share @@ -323,11 +329,16 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, * so each graph has its independent data paths, but multi * CFs may share same optimized disk(or optimized disk path). */ - sessions = origin.copy(config, this.database, this.store); + // Keep checking, copying and opening atomic with native restore. + if (origin.databaseOpened()) { + sessions = origin.copy(config, this.database, this.store); + return this.registerOpenedSessions(dataPath, sessions); + } } } - if (e.getMessage().contains("Column family not found")) { + if (!(e instanceof RocksDBStdSessions.RecoveryLockException) && + e.getMessage().contains("Column family not found")) { if (this.isSchemaStore()) { LOG.info("Failed to open RocksDB '{}' with database '{}'," + " try to init CF later", dataPath, this.database); @@ -360,19 +371,21 @@ protected RocksDBSessions open(HugeConfig config, String dataPath, } } - if (sessions != null) { - // May override the original session pool - this.dbs.put(dataPath, sessions); - sessions.session().open(); - LOG.debug("Store opened: {}", dataPath); - } + return this.registerOpenedSessions(dataPath, sessions); + } + private RocksDBSessions registerOpenedSessions(String dataPath, RocksDBSessions sessions) { + sessions.session().open(); + // May override the original session pool after the new session opened. + this.dbs.put(dataPath, sessions); + LOG.debug("Store opened: {}", dataPath); return sessions; } protected RocksDBSessions openSessionPool(HugeConfig config, String dataPath, String walPath, - List tableNames) throws RocksDBException { + List tableNames) throws + RocksDBException { if (tableNames == null) { return new RocksDBStdSessions(config, this.database, this.store, dataPath, walPath); } else { @@ -417,7 +430,7 @@ public void close() { @Override public boolean opened() { this.checkDbOpened(); - return this.sessions.session().opened(); + return this.sessions.databaseOpened(); } @Override @@ -630,8 +643,12 @@ public synchronized void truncate() { try { this.checkOpened(); - this.clear(false); - this.init(); + if (this.toplingProvider) { + this.clearTables(); + } else { + this.clear(false); + this.init(); + } // Clear write-batch this.dbs.values().forEach(BackendSessionPool::forceResetSessions); LOG.debug("Store truncated: {}", this.store); @@ -640,6 +657,22 @@ public synchronized void truncate() { } } + private void clearTables() { + this.sessions.clearTables(this.tableNames()); + + Map tableDBMap = this.tableDBMapping(); + for (Map.Entry entry : + tableDBMap.entrySet()) { + Collection tables; + if (entry.getKey().equals(HugeType.OLAP.string())) { + tables = this.olapTables(); + } else { + tables = Collections.singletonList(entry.getKey()); + } + entry.getValue().clearTables(tables); + } + } + @Override public void beginTx() { Lock readLock = this.storeLock.readLock(); @@ -718,8 +751,8 @@ public Map createSnapshot(String snapshotPrefix) { @Override public void resumeSnapshot(String snapshotPrefix, boolean deleteSnapshot) { - Lock readLock = this.storeLock.readLock(); - readLock.lock(); + Lock writeLock = this.storeLock.writeLock(); + writeLock.lock(); try { if (!this.opened()) { return; @@ -754,7 +787,7 @@ public void resumeSnapshot(String snapshotPrefix, boolean deleteSnapshot) { } catch (RocksDBException | IOException e) { throw new BackendException("Failed to resume snapshot", e); } finally { - readLock.unlock(); + writeLock.unlock(); } } diff --git a/hugegraph-server/hugegraph-test/README.md b/hugegraph-server/hugegraph-test/README.md new file mode 100644 index 0000000000..5abe65c382 --- /dev/null +++ b/hugegraph-server/hugegraph-test/README.md @@ -0,0 +1,20 @@ +# Server integration tests + +Run API tests against a dedicated test graph service with the matching backend +profile, for example `mvn test -pl hugegraph-server/hugegraph-test -am -P api-test,hstore`. +The suite creates and removes schemas, data, users and graphspaces. Its default +graph is `DEFAULT/hugegraph`; use an isolated service and data volumes. + +`BaseApiTest` accepts the following optional environment variables so the same +suite can run through a port-forward against a Helm-managed test deployment: + +| Variable | Default | +| --- | --- | +| `HUGEGRAPH_TEST_SERVER_URL` | `http://127.0.0.1:8080` | +| `HUGEGRAPH_TEST_ADMIN_USERNAME` | `admin` | +| `HUGEGRAPH_TEST_ADMIN_PASSWORD` | Existing local test fixture password | + +Supply credentials through the test process environment. Do not place them in +Maven command-line arguments or save them to test evidence. The overrides affect +the administrator client; per-test users still use their own fixture credentials. +Existing local and CI invocations keep their defaults when variables are absent. diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/HugeFactoryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/HugeFactoryTest.java new file mode 100644 index 0000000000..101ab77860 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/HugeFactoryTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph; + +import java.util.Arrays; + +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class HugeFactoryTest { + + @Test + public void testCleanupContinuesAfterGraphFailure() { + StandardHugeGraph failed = Mockito.mock(StandardHugeGraph.class); + StandardHugeGraph cleaned = Mockito.mock(StandardHugeGraph.class); + Mockito.doThrow(new HugeException("test")) + .when(failed).closeCurrentThreadTransaction(); + + Assert.assertThrows(HugeException.class, () -> { + HugeFactory.closeCurrentThreadTransactions( + Arrays.asList(failed, cleaned)); + }); + + Mockito.verify(failed).closeCurrentThreadTransaction(); + Mockito.verify(cleaned).closeCurrentThreadTransaction(); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java index abe4b6839c..2b5d9aa129 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/BaseApiTest.java @@ -59,13 +59,16 @@ public class BaseApiTest { - protected static final String BASE_URL = "http://127.0.0.1:8080"; + protected static final String BASE_URL = System.getenv().getOrDefault( + "HUGEGRAPH_TEST_SERVER_URL", "http://127.0.0.1:8080"); private static final String GRAPH = "hugegraph"; private static final String GRAPHSPACE = "DEFAULT"; protected static final String URL_PREFIX = "graphspaces/" + GRAPHSPACE + "/graphs/" + GRAPH; protected static final String TRAVERSERS_API = URL_PREFIX + "/traversers"; - private static final String USERNAME = "admin"; - private static final String PASSWORD = "pa"; + private static final String USERNAME = System.getenv().getOrDefault( + "HUGEGRAPH_TEST_ADMIN_USERNAME", "admin"); + private static final String PASSWORD = System.getenv().getOrDefault( + "HUGEGRAPH_TEST_ADMIN_PASSWORD", "pa"); private static final int NO_LIMIT = -1; private static final String SCHEMA_PKS = "/schema/propertykeys"; private static final String SCHEMA_VLS = "/schema/vertexlabels"; @@ -181,12 +184,21 @@ protected static void waitTaskStatus(int task, Set expectedStatus) { String.valueOf(task)); String content = assertResponseStatus(200, r); status = assertJsonContains(content, "task_status"); + if (expectedStatus.contains(status)) { + return; + } + // A success-only wait must not spin after the task has ended. + if (!expectedStatus.contains("failed") && + ("failed".equals(status) || "cancelled".equals(status))) { + Assert.fail(String.format("Task %s ended with status %s: %s", + task, status, content)); + } if (times++ > maxTimes) { Assert.fail(String.format("Failed to wait for task %s " + - "due to timeout", task)); + "due to timeout, last status %s", + task, status)); } - } - while (!expectedStatus.contains(status)); + } while (true); } protected static void initVertexLabel() { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java index 98240e76f5..c8ded372d0 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GraphSpaceApiTest.java @@ -24,6 +24,7 @@ import org.apache.hugegraph.util.JsonUtil; import org.junit.Assert; import org.junit.Assume; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -51,6 +52,32 @@ public void removeSpaces() { this.client().delete(PATH, space); } } + this.clearRoleTestFixtures(); + } + + @After + public void clearRoleTestFixtures() { + if (!"hstore".equals(System.getProperty("backend"))) { + return; + } + String space = "default_role_auth_space"; + Response spacesResponse = this.client().get(PATH); + Map spaces = JsonUtil.fromJson( + assertResponseStatus(200, spacesResponse), Map.class); + if (((List) spaces.get("graphSpaces")).contains(space)) { + assertResponseStatus(204, this.client().delete(PATH, space)); + } + String usersPath = "graphspaces/DEFAULT/auth/users"; + Response usersResponse = this.client().get(usersPath, ImmutableMap.of("limit", -1)); + Map result = JsonUtil.fromJson( + assertResponseStatus(200, usersResponse), Map.class); + List ownedNames = List.of("default_role_analyst", "default_role_target", + "default_role_manager"); + for (Map user : (List>) result.get("users")) { + if (ownedNames.contains(user.get("user_name"))) { + assertResponseStatus(204, this.client().delete(usersPath, (String) user.get("id"))); + } + } } @Test @@ -505,49 +532,60 @@ public void testDefaultRoleMutationRequiresSpaceManager() { String manager = "default_role_manager"; createSpace(space, true); - RestClient analystClient = userClient(analyst); - userClient(target); - RestClient managerClient = spaceManagerClient(space, manager); - - String rolePath = String.format("graphspaces/%s/role", space); - String analystBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", - analyst); - Response r = this.client().post(rolePath, analystBody); - assertResponseStatus(201, r); - - String grantTargetBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", - target); - r = analystClient.post(rolePath, grantTargetBody); - assertResponseStatus(403, r); - - r = analystClient.get(rolePath, - ImmutableMap.of("user", target, - "role", "ANALYST")); - assertResponseStatus(403, r); - - r = analystClient.delete(rolePath, - ImmutableMap.of("user", analyst, - "role", "ANALYST")); - assertResponseStatus(403, r); - - r = managerClient.post(rolePath, grantTargetBody); - assertResponseStatus(201, r); - - r = managerClient.get(rolePath, - ImmutableMap.of("user", target, - "role", "ANALYST")); - String result = assertResponseStatus(200, r); - Assert.assertTrue(result.contains("true")); - - String spaceRoleBody = String.format("{\"user\":\"%s\",\"role\":\"SPACE\"}", - target); - r = managerClient.post(rolePath, spaceRoleBody); - assertResponseStatus(403, r); - - r = managerClient.delete(rolePath, - ImmutableMap.of("user", target, - "role", "ANALYST")); - assertResponseStatus(204, r); + RestClient analystClient = null; + RestClient managerClient = null; + try { + analystClient = userClient(analyst); + userClient(target).close(); + managerClient = spaceManagerClient(space, manager); + + String rolePath = String.format("graphspaces/%s/role", space); + String analystBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", + analyst); + Response r = this.client().post(rolePath, analystBody); + assertResponseStatus(201, r); + + String grantTargetBody = String.format("{\"user\":\"%s\",\"role\":\"ANALYST\"}", + target); + r = analystClient.post(rolePath, grantTargetBody); + assertResponseStatus(403, r); + + r = analystClient.get(rolePath, + ImmutableMap.of("user", target, + "role", "ANALYST")); + assertResponseStatus(403, r); + + r = analystClient.delete(rolePath, + ImmutableMap.of("user", analyst, + "role", "ANALYST")); + assertResponseStatus(403, r); + + r = managerClient.post(rolePath, grantTargetBody); + assertResponseStatus(201, r); + + r = managerClient.get(rolePath, + ImmutableMap.of("user", target, + "role", "ANALYST")); + String result = assertResponseStatus(200, r); + Assert.assertTrue(result.contains("true")); + + String spaceRoleBody = String.format("{\"user\":\"%s\",\"role\":\"SPACE\"}", + target); + r = managerClient.post(rolePath, spaceRoleBody); + assertResponseStatus(403, r); + + r = managerClient.delete(rolePath, + ImmutableMap.of("user", target, + "role", "ANALYST")); + assertResponseStatus(204, r); + } finally { + if (analystClient != null) { + analystClient.close(); + } + if (managerClient != null) { + managerClient.close(); + } + } } @Test diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/ContextTaskTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/ContextTaskTest.java new file mode 100644 index 0000000000..a5ef4ad8d7 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/ContextTaskTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.auth; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.auth.HugeGraphAuthProxy.Context; +import org.apache.hugegraph.auth.HugeGraphAuthProxy.ContextTask; +import org.apache.hugegraph.testutil.Assert; +import org.junit.After; +import org.junit.Test; + +public class ContextTaskTest { + + @After + public void teardown() { + HugeGraphAuthProxy.resetContext(); + } + + @Test + public void testCleanupRunsBeforeContextReset() { + Context context = Context.admin(); + AtomicBoolean runnerCalled = new AtomicBoolean(false); + AtomicBoolean cleanupCalled = new AtomicBoolean(false); + HugeGraphAuthProxy.setContext(context); + + ContextTask task = new ContextTask(() -> { + Assert.assertSame(context, HugeGraphAuthProxy.getContext()); + runnerCalled.set(true); + }, () -> { + Assert.assertSame(context, HugeGraphAuthProxy.getContext()); + cleanupCalled.set(true); + }); + HugeGraphAuthProxy.resetContext(); + + task.run(); + + Assert.assertTrue(runnerCalled.get()); + Assert.assertTrue(cleanupCalled.get()); + Assert.assertNull(HugeGraphAuthProxy.getContext()); + } + + @Test + public void testCleanupRunsWhenTaskFails() { + AtomicBoolean cleanupCalled = new AtomicBoolean(false); + ContextTask task = new ContextTask(() -> { + throw new IllegalStateException("test"); + }, () -> cleanupCalled.set(true)); + + Assert.assertThrows(IllegalStateException.class, task::run); + Assert.assertTrue(cleanupCalled.get()); + Assert.assertNull(HugeGraphAuthProxy.getContext()); + } + + @Test + public void testContextResetRunsWhenCleanupFails() { + Context context = Context.admin(); + HugeGraphAuthProxy.setContext(context); + ContextTask task = new ContextTask(() -> { + Assert.assertSame(context, HugeGraphAuthProxy.getContext()); + }, () -> { + throw new IllegalStateException("test"); + }); + HugeGraphAuthProxy.resetContext(); + + task.run(); + + Assert.assertNull(HugeGraphAuthProxy.getContext()); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSnapshotRestoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSnapshotRestoreTest.java new file mode 100644 index 0000000000..f9184dd9d5 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBSnapshotRestoreTest.java @@ -0,0 +1,729 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.backend.store.rocksdb; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.channels.FileChannel; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.hugegraph.backend.BackendException; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.exception.ConnectionException; +import org.apache.hugegraph.unit.FakeObjects; +import org.apache.hugegraph.testutil.Whitebox; +import org.junit.Rule; +import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.rocksdb.Status; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class RocksDBSnapshotRestoreTest { + + @Rule + public TemporaryFolder temporary = new TemporaryFolder(); + + @Test + public void testDataCopyFailureRetainsCheckpointAndRetries() throws Exception { + this.failureAndRetry("data-copy", "separate"); + } + + @Test + public void testWalCopyFailureRetainsCheckpointAndRetries() throws Exception { + this.failureAndRetry("copy", "separate"); + } + + @Test + public void testShortCopyIsNotPublished() throws Exception { + this.failureAndRetry("short-copy", "separate"); + } + + @Test + public void testSameLengthCorruptionIsNotPublished() throws Exception { + this.failureAndRetry("corrupt-copy", "separate"); + } + + @Test + public void testRetirementFailureRetainsMarkerAndRetries() throws Exception { + this.failureAndRetry("retire", "separate"); + } + + @Test + public void testPublishFailureRetainsMarkerAndRetries() throws Exception { + this.failureAndRetry("publish", "separate"); + } + + @Test + public void testParentWalRetirementFailurePreservesData() throws Exception { + this.failureAndRetry("retire", "parent"); + } + + @Test + public void testNestedWalPublishFailureRetries() throws Exception { + this.failureAndRetry("publish", "nested"); + } + + @Test + public void testSymlinkWalPublishFailurePreservesLink() throws Exception { + this.failureAndRetry("publish", "symlink"); + } + + @Test + public void testSymlinkInsideDataSurvivesInterruptedRestore() throws Exception { + this.failureAndRetry("data-copy", "nested-symlink"); + } + + @Test + public void testIndirectSymlinkInsideDataSurvivesInterruptedRestore() throws Exception { + this.failureAndRetry("data-copy", "ancestor-symlink"); + } + + @Test + public void testCorruptedDataCopyIsNotOpened() throws Exception { + this.failureAndRetry("corrupt-data-copy", "separate"); + } + + private static void fakeCheckpoint(File snapshot) throws IOException { + Files.write(new File(snapshot, "CURRENT").toPath(), "MANIFEST-000001\n".getBytes(StandardCharsets.UTF_8)); + Files.write(new File(snapshot, "MANIFEST-000001").toPath(), new byte[]{1, 2, 3}); + } + + private void failureAndRetry(String fault, String layout) throws Exception { + File root = this.temporary.newFolder(); + File data = new File(root, "data"); + File wal = new File(root, "wal"); + File snapshot = new File(root, "snapshot"); + if ("parent".equals(layout)) { + data = new File(wal, "data"); + snapshot = new File(wal, "checkpoints/snapshot"); + } else if ("nested".equals(layout)) { + wal = new File(data, "wal"); + } + FileUtils.forceMkdir(data); + if (layout.contains("symlink")) { + File target = new File(root, "target"); + FileUtils.forceMkdir(target); + File linkParent = "nested-symlink".equals(layout) || "ancestor-symlink".equals(layout) ? data : root; + wal = new File(linkParent, "wal-link"); + Files.createSymbolicLink(wal.toPath(), target.toPath()); + if ("ancestor-symlink".equals(layout)) { + wal = new File(wal, "child-wal"); + FileUtils.forceMkdir(wal); + } + } else { + FileUtils.forceMkdir(wal); + } + FileUtils.forceMkdir(snapshot); + fakeCheckpoint(snapshot); + byte[] tail = "checkpoint".getBytes(StandardCharsets.UTF_8); + FileUtils.writeByteArrayToFile(new File(snapshot, "000123.log"), tail); + FileUtils.writeByteArrayToFile(new File(snapshot, "keep.txt"), tail); + // Same-name, same-length old WAL must never be accepted as the checkpoint. + FileUtils.writeByteArrayToFile(new File(wal, "000123.log"), + "old-oldwal".getBytes(StandardCharsets.UTF_8)); + FileUtils.writeByteArrayToFile(new File(wal, "000124.log"), tail); + RocksDBSnapshotRestore restore = new RocksDBSnapshotRestore( + data.toString(), wal.toString(), snapshot.toString(), new FaultyFiles(fault)); + restore.begin(); + try { + restore.install(); + fail("Expected injected " + fault); + } catch (IOException expected) { + assertTrue(new File(data + ".resume-pending").isFile()); + assertArrayEquals(tail, Files.readAllBytes(new File(snapshot, "000123.log").toPath())); + } + // The public resume path can retry the SAME source even if its WAL alias + // disappeared between deleting data and copying the checkpoint. + RocksDBSnapshotRestore.start(data.toString(), wal.toString(), snapshot.toString()); + RocksDBSnapshotRestore retry = RocksDBSnapshotRestore.prepareOpen(data.toString(), wal.toString()); + assertNotNull(retry); + assertArrayEquals(tail, Files.readAllBytes(new File(wal, "000123.log").toPath())); + assertFalse(new File(wal, "000124.log").exists()); + assertFalse(new File(data, "000123.log").exists()); + assertTrue(new File(data, "keep.txt").isFile()); + assertTrue(new File(snapshot, "000123.log").isFile()); + if (layout.contains("symlink")) { + assertTrue(Files.isSymbolicLink("ancestor-symlink".equals(layout) ? + wal.getParentFile().toPath() : wal.toPath())); + } + retry.complete(); + assertFalse(new File(data + ".resume-pending").exists()); + } + + @Test + public void testWalNameCollisionFailsClosed() throws Exception { + File data = this.temporary.newFolder("data"); + File wal = this.temporary.newFolder("wal"); + File snapshot = this.temporary.newFolder("snapshot"); + fakeCheckpoint(snapshot); + FileUtils.writeByteArrayToFile(new File(snapshot, "000123.log"), new byte[]{1}); + FileUtils.forceMkdir(new File(wal, "000123.log")); + RocksDBSnapshotRestore restore = new RocksDBSnapshotRestore( + data.toString(), wal.toString(), snapshot.toString(), + new RocksDBSnapshotRestore.FileOperations()); + restore.begin(); + try { + RocksDBSnapshotRestore.prepareOpen(data.toString(), wal.toString()); + fail("WAL directory collision must fail closed"); + } catch (BackendException expected) { + assertTrue(new File(data + ".resume-pending").isFile()); + assertTrue(new File(snapshot, "000123.log").isFile()); + } + } + + @Test + public void testFreshSessionRecoversPendingSnapshotBeforeNativeOpen() throws Exception { + File data = this.temporary.newFolder("data"); + File wal = this.temporary.newFolder("wal"); + File snapshot = new File(this.temporary.newFolder("snapshot-root"), "rocks"); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions original = new RocksDBStdSessions(config, "db", "store", data.toString(), wal.toString()); + byte[] key = new byte[]{1}; + try { + original.createTable("test"); + original.session().put("test", key, new byte[]{2}); + original.session().commit(); + original.createSnapshot(snapshot.toString()); + original.session().put("test", key, new byte[]{3}); + original.session().commit(); + } finally { + original.close(); + } + RocksDBSnapshotRestore restore = new RocksDBSnapshotRestore( + data.toString(), wal.toString(), snapshot.toString(), new FaultyFiles("data-copy")); + restore.begin(); + try { + restore.install(); + fail("Expected data copy failure"); + } catch (IOException expected) { + assertTrue(new File(data + ".resume-pending").exists()); + } + RocksDBSessions fresh = new RocksDBStdSessions(config, "db", "store", data.toString(), wal.toString(), + Collections.singletonList("test")); + try { + assertArrayEquals(new byte[]{2}, fresh.session().get("test", key)); + assertFalse(new File(data + ".resume-pending").exists()); + assertFalse(snapshot.exists()); + } finally { + fresh.close(); + } + } + + @Test + public void testBothOpenOverloadsRejectIncompleteMarker() throws Exception { + File data = this.temporary.newFolder("data"); + File marker = new File(data + ".resume-pending"); + Files.write(marker.toPath(), new byte[0]); + HugeConfig config = FakeObjects.newConfig(); + for (boolean withTables : new boolean[]{false, true}) { + try { + if (withTables) { + new RocksDBStdSessions(config, "db", "store", data.toString(), data.toString(), + Collections.emptyList()); + } else { + new RocksDBStdSessions(config, "db", "store", data.toString(), data.toString()); + } + fail("Native open must not bypass incomplete marker"); + } catch (BackendException expected) { + assertEquals(0, data.list().length); + assertTrue(marker.isFile()); + } + } + } + + @Test + public void testCompetingRecoveryCannotEnterWhileOwnerHoldsLock() throws Exception { + File data = this.temporary.newFolder("data"); + try (FileChannel owner = RocksDBSnapshotRestore.lock(data.toString())) { + CountDownLatch done = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + Thread contender = new Thread(() -> { + try { + for (boolean withTables : new boolean[]{false, true}) { + try { + if (withTables) { + new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString(), + Collections.emptyList()); + } else { + new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + } + fail("Competing open acquired recovery lock"); + } catch (RocksDBException expected) { + assertTrue(expected.getMessage().contains("No locks available")); + assertEquals(Status.Code.IOError, expected.getStatus().getCode()); + } + } + } catch (Throwable e) { + failure.set(e); + } finally { + done.countDown(); + } + }); + contender.start(); + assertTrue(done.await(10, TimeUnit.SECONDS)); + if (failure.get() != null) { + throw new AssertionError(failure.get()); + } + assertEquals(0, data.list().length); + } + try (FileChannel retry = RocksDBSnapshotRestore.lock(data.toString())) { + assertTrue(retry.isOpen()); + } + } + + @Test + public void testGuardErrorTextCannotReuseCachedNativeOwner() throws Exception { + File data = this.temporary.newFolder("No locks available Column family not found"); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions owner = new RocksDBStdSessions(config, "db", "store", + data.toString(), data.toString()); + RocksDBStore store = new RocksDBStore.RocksDBGraphStore(new RocksDBStoreProvider(), "db", "store"); + Map databases = Whitebox.getInternalState(store, "dbs"); + databases.put(data.toString(), owner); + Path guard = new File(data + ".resume-lock").toPath(); + Path held = new File(data + ".owned-lock").toPath(); + try { + owner.createTable("test"); + owner.session().put("test", new byte[]{1}, new byte[]{2}); + owner.session().commit(); + // Genuine contention can share the live owner's CFs. + RocksDBSessions copy = store.open(config, data.toString(), data.toString(), + Collections.singletonList("test")); + assertArrayEquals(new byte[]{2}, copy.session().get("test", new byte[]{1})); + copy.close(); + databases.put(data.toString(), owner); + + // Inject an IO failure, retaining the locked inode and live native owner. + Files.move(guard, held); + Files.createDirectory(guard); + try { + store.open(config, data.toString(), data.toString(), Collections.singletonList("test")); + fail("Path text must not classify a guard IO failure as contention or missing CF"); + } catch (ConnectionException expected) { + assertTrue(expected.getCause() instanceof RocksDBStdSessions.RecoveryLockException); + assertFalse(((RocksDBStdSessions.RecoveryLockException) expected.getCause()).isContention()); + assertTrue(expected.getCause().getMessage().contains("No locks available")); + assertTrue(expected.getCause().getMessage().contains("Column family not found")); + assertTrue(owner.databaseOpened()); + assertArrayEquals(new byte[]{2}, owner.session().get("test", new byte[]{1})); + assertFalse(new File(data + ".resume-pending").exists()); + } + } finally { + if (Files.exists(held)) { + Files.delete(guard); + Files.move(held, guard); + } + owner.forceCloseRocksDB(); + } + } + + @Test + public void testLiveDatabasePreventsPendingRecovery() throws Exception { + File data = this.temporary.newFolder("data"); + File snapshot = new File(this.temporary.newFolder("snapshot-root"), "rocks"); + RocksDBSessions owner = new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + try { + owner.createSnapshot(snapshot.toString()); + RocksDBSnapshotRestore restore = new RocksDBSnapshotRestore( + data.toString(), data.toString(), snapshot.toString(), + new RocksDBSnapshotRestore.FileOperations()); + restore.begin(); + byte[] manifest = Files.readAllBytes(new File(data, "CURRENT").toPath()); + try { + new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + fail("Must not restore under an open native owner"); + } catch (RocksDBException expected) { + assertTrue(expected.getMessage().contains("No locks available")); + assertEquals(Status.Code.IOError, expected.getStatus().getCode()); + assertArrayEquals(manifest, Files.readAllBytes(new File(data, "CURRENT").toPath())); + assertTrue(owner.databaseOpened()); + assertTrue(snapshot.isDirectory()); + } + } finally { + owner.forceCloseRocksDB(); + } + RocksDBSessions recovered = new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + recovered.forceCloseRocksDB(); + assertFalse(new File(data + ".resume-pending").exists()); + } + + @Test + public void testRepeatedAdapterSnapshotResumePreservesConsumeSemantics() throws Exception { + File root = this.temporary.newFolder("adapter"); + // createCheckpoint's existing assertion requires a snapshot-prefixed parent. + File data = new File(root, "snapshot-data/store"); + File wal = new File(root, "wal"); + RocksDBSessions sessions = new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), wal.toString()); + RocksDBStore store = new RocksDBStore.RocksDBGraphStore(new RocksDBStoreProvider(), "db", "store"); + Whitebox.setInternalState(store, "sessions", sessions); + Map databases = Whitebox.getInternalState(store, "dbs"); + databases.put(data.toString(), sessions); + byte[] key = new byte[]{1}; + try { + sessions.createTable("test"); + sessions.session().put("test", key, new byte[]{2}); + sessions.session().commit(); + store.createSnapshot("snapshot"); + String snapshot = sessions.buildSnapshotPath("snapshot"); + for (boolean consume : new boolean[]{false, false, true}) { + sessions.session().put("test", key, new byte[]{3}); + sessions.session().commit(); + store.resumeSnapshot("snapshot", consume); + assertArrayEquals(new byte[]{2}, sessions.session().get("test", key)); + assertEquals(!consume, new File(snapshot).exists()); + assertFalse(new File(data + "_temp").exists()); + assertFalse(new File(data + ".resume-pending").exists()); + } + } finally { + sessions.close(); + } + } + + @Test + public void testMissingManifestRejectedBeforeDataMutation() throws Exception { + File data = this.temporary.newFolder("data"); + File snapshot = this.temporary.newFolder("snapshot"); + File sentinel = new File(data, "live"); + Files.write(sentinel.toPath(), new byte[]{42}); + try { + RocksDBSnapshotRestore.start(data.toString(), data.toString(), snapshot.toString()); + fail("Incomplete snapshot must not be installed"); + } catch (IOException expected) { + assertArrayEquals(new byte[]{42}, Files.readAllBytes(sentinel.toPath())); + assertFalse(new File(data + ".resume-pending").exists()); + } + } + + @Test + public void testNativeOpenFailureKeepsRecoverySourceAndMarker() throws Exception { + File data = this.temporary.newFolder("data"); + File snapshot = this.temporary.newFolder("snapshot"); + fakeCheckpoint(snapshot); + RocksDBSnapshotRestore.start(data.toString(), data.toString(), snapshot.toString()); + try { + new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", data.toString(), data.toString()); + fail("Invalid native manifest must not report success"); + } catch (org.rocksdb.RocksDBException expected) { + assertTrue(new File(data + ".resume-pending").isFile()); + assertArrayEquals(new byte[]{1, 2, 3}, + Files.readAllBytes(new File(snapshot, "MANIFEST-000001").toPath())); + } + } + + @Test + public void testRecoveryLockExcludesAnotherProcess() throws Exception { + File data = this.temporary.newFolder("data"); + String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + try (FileChannel owner = RocksDBSnapshotRestore.lock(data.toString())) { + Process child = new ProcessBuilder(new File(System.getProperty("java.home"), "bin/java").toString(), + "-cp", classpath, getClass().getName(), data.toString()) + .redirectErrorStream(true).start(); + try { + assertTrue("Competing process did not finish", child.waitFor(10, TimeUnit.SECONDS)); + String output = new String(child.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertEquals(output, 23, child.exitValue()); + assertEquals(0, data.list().length); + } finally { + child.destroyForcibly(); + } + } + } + + @Test + public void testExternalOwnerRejectsBothPublicOpenPaths() throws Exception { + File data = this.temporary.newFolder("external-owner"); + File ready = new File(this.temporary.getRoot(), "owner-ready"); + String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + Process child = new ProcessBuilder(new File(System.getProperty("java.home"), "bin/java").toString(), + "-cp", classpath, getClass().getName(), data.toString(), + ready.toString()).redirectErrorStream(true).start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!ready.exists() && child.isAlive() && System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertTrue("External owner did not acquire lock", ready.exists()); + for (boolean withTables : new boolean[]{false, true}) { + try { + if (withTables) { + new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString(), Collections.emptyList()); + } else { + new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + } + fail("Public open must reject a competing process before native recovery"); + } catch (RocksDBException expected) { + assertTrue(expected instanceof RocksDBStdSessions.RecoveryLockException); + assertTrue(((RocksDBStdSessions.RecoveryLockException) expected).isContention()); + assertEquals(Status.Code.IOError, expected.getStatus().getCode()); + assertTrue(expected.getMessage().contains("No locks available")); + assertEquals(0, data.list().length); + } + } + child.getOutputStream().write('\n'); + child.getOutputStream().flush(); + assertTrue("External owner did not close", child.waitFor(10, TimeUnit.SECONDS)); + assertEquals(0, child.exitValue()); + } finally { + child.destroyForcibly(); + } + try (FileChannel retry = RocksDBSnapshotRestore.lock(data.toString())) { + assertTrue(retry.isOpen()); + } + } + + public static void main(String[] args) { + try (FileChannel channel = RocksDBSnapshotRestore.lock(args[0])) { + if (args.length == 2) { + Files.write(new File(args[1]).toPath(), new byte[]{1}); + System.in.read(); + } else { + System.out.println("acquired"); + } + } catch (BackendException expected) { + System.exit(23); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Test + public void testCachedOwnerCopyCannotOverlapNativeReload() throws Exception { + File data = this.temporary.newFolder("copy-reload"); + CountDownLatch checked = new CountDownLatch(1); + CountDownLatch allowCopy = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + AtomicReference copied = new AtomicReference<>(); + RocksDBStdSessions owner = new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()) { + @Override + public boolean databaseOpened() { + boolean opened = super.databaseOpened(); + checked.countDown(); + try { + assertTrue("Copy check was not released", allowCopy.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + return opened; + } + + @Override + public RocksDBSessions copy(HugeConfig config, String database, String store) { + assertTrue("Native owner closed between its open check and copy", super.databaseOpened()); + return super.copy(config, database, store); + } + }; + RocksDBStore store = new RocksDBStore.RocksDBGraphStore(new RocksDBStoreProvider(), "db", "store"); + Map databases = Whitebox.getInternalState(store, "dbs"); + databases.put(data.toString(), owner); + Thread open = new Thread(() -> { + try { + copied.set(store.open(FakeObjects.newConfig(), data.toString(), data.toString(), + Collections.singletonList("test"))); + } catch (Throwable e) { + failure.compareAndSet(null, e); + } + }, "cached-owner-copy"); + Thread reload = new Thread(() -> { + try { + owner.reloadRocksDB(); + } catch (Throwable e) { + failure.compareAndSet(null, e); + } + }, "cached-owner-reload"); + try { + owner.createTable("test"); + owner.session().put("test", new byte[]{1}, new byte[]{2}); + owner.session().commit(); + open.start(); + assertTrue("Open did not reach the cached owner check", checked.await(10, TimeUnit.SECONDS)); + reload.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (reload.getState() != Thread.State.BLOCKED && reload.isAlive() && + System.nanoTime() < deadline) { + Thread.sleep(10); + } + assertEquals("Native reload must wait for the owner check and copy", Thread.State.BLOCKED, + reload.getState()); + allowCopy.countDown(); + open.join(10000); + reload.join(10000); + assertFalse(open.isAlive()); + assertFalse(reload.isAlive()); + if (failure.get() != null) { + throw new AssertionError(failure.get()); + } + assertNotNull(copied.get()); + assertArrayEquals(new byte[]{2}, owner.session().get("test", new byte[]{1})); + } finally { + allowCopy.countDown(); + open.join(10000); + reload.join(10000); + owner.forceCloseRocksDB(); + } + } + + @Test + public void testNativeCloseToRestoreTransitionRetainsSameLock() throws Exception { + File data = this.temporary.newFolder("data"); + File snapshot = new File(this.temporary.newFolder("snapshot-root"), "rocks"); + RocksDBSessions sessions = new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + try { + sessions.createTable("test"); + sessions.session().put("test", new byte[]{1}, new byte[]{2}); + sessions.session().commit(); + sessions.createSnapshot(snapshot.toString()); + OpenedRocksDB old = Whitebox.getInternalState(sessions, "rocksdb"); + FileChannel held = old.closeForRestore(); + assertFalse(old.isOwningHandle()); + assertTrue(held.isOpen()); + RocksDBStore store = new RocksDBStore.RocksDBGraphStore(new RocksDBStoreProvider(), "db", "store"); + Map databases = Whitebox.getInternalState(store, "dbs"); + databases.put(data.toString(), sessions); + try { + store.open(FakeObjects.newConfig(), data.toString(), data.toString(), + Collections.singletonList("test")); + fail("Cached owner must not be copied while native close/restore is in progress"); + } catch (ConnectionException expected) { + assertFalse(sessions.databaseOpened()); + assertTrue(held.isOpen()); + } + // Deterministic observation of the exact native-closed/marker-not-yet-created window. + try { + new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + fail("Native close must not release recovery ownership"); + } catch (RocksDBException expected) { + assertTrue(expected.getMessage().contains("No locks available")); + assertEquals(Status.Code.IOError, expected.getStatus().getCode()); + assertFalse(new File(data + ".resume-pending").exists()); + } + RocksDBSnapshotRestore.start(data.toString(), data.toString(), snapshot.toString()); + sessions.reloadRocksDB(); + OpenedRocksDB replacement = Whitebox.getInternalState(sessions, "rocksdb"); + assertSame(held, Whitebox.getInternalState(replacement, "recoveryLock")); + assertArrayEquals(new byte[]{2}, sessions.session().get("test", new byte[]{1})); + } finally { + sessions.close(); + } + try (FileChannel available = RocksDBSnapshotRestore.lock(data.toString())) { + assertTrue(available.isOpen()); + } + } + + @Test + public void testForceCloseReleasesNativeAndRecoveryOwnership() throws Exception { + File data = this.temporary.newFolder("data"); + RocksDBSessions sessions = new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + sessions.forceCloseRocksDB(); + assertFalse(sessions.databaseOpened()); + RocksDBSessions fresh = new RocksDBStdSessions(FakeObjects.newConfig(), "db", "store", + data.toString(), data.toString()); + fresh.forceCloseRocksDB(); + } + + @Test + public void testRootAndLinuxMountPointRejectedBeforeCreatingRecoveryLock() { + try { + RocksDBSnapshotRestore.lock(File.listRoots()[0].toString()); + fail("Filesystem root must not be a store directory"); + } catch (BackendException expected) { + assertTrue(expected.getCause().getMessage().contains("filesystem root")); + } + if (System.getProperty("os.name").startsWith("Linux")) { + Path lock = new File("/proc.resume-lock").toPath(); + assertFalse(Files.exists(lock)); + try { + RocksDBSnapshotRestore.lock("/proc"); + fail("Linux mount point must not be a store directory"); + } catch (BackendException expected) { + assertTrue(expected.getCause().getMessage().contains("mount point")); + } + assertFalse(Files.exists(lock)); + } + } + + private static class FaultyFiles extends RocksDBSnapshotRestore.FileOperations { + + private final String fault; + + FaultyFiles(String fault) { + this.fault = fault; + } + + @Override + void copyDirectory(File source, File target) throws IOException { + if ("data-copy".equals(this.fault)) { + throw new IOException("injected data copy failure"); + } + super.copyDirectory(source, target); + if ("corrupt-data-copy".equals(this.fault)) { + Files.write(new File(target, "MANIFEST-000001").toPath(), new byte[]{3, 2, 1}); + } + } + + @Override + void copyFile(File source, File target) throws IOException { + if ("copy".equals(this.fault)) { + throw new IOException("injected copy failure"); + } + super.copyFile(source, target); + if ("short-copy".equals(this.fault)) { + Files.write(target.toPath(), new byte[]{0}); + } else if ("corrupt-copy".equals(this.fault)) { + Files.write(target.toPath(), new byte[(int) target.length()]); + } + } + + @Override + void move(Path source, Path target) throws IOException { + if (("retire".equals(this.fault) && target.toString().endsWith(".aside")) || + ("publish".equals(this.fault) && target.toString().endsWith(".log"))) { + throw new IOException("injected " + this.fault + " failure"); + } + super.move(source, target); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphTransactionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphTransactionTest.java index 3cb79e3687..b7f4bf2820 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphTransactionTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/backend/tx/GraphTransactionTest.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.apache.hugegraph.HugeFactory; @@ -49,6 +50,21 @@ public class GraphTransactionTest { + @Test + public void testOlapDeletionDoesNotScanBaseVertexEdges() { + GraphTransaction transaction = Mockito.mock(GraphTransaction.class, Mockito.CALLS_REAL_METHODS); + HugeVertex vertex = Mockito.mock(HugeVertex.class); + Mockito.when(vertex.schemaLabel()).thenReturn(VertexLabel.OLAP_VL); + Id id = IdGenerator.of(1L); + + // Preserve OLAP's original rejection before any ordinary edge scan. + Assert.assertThrows(IllegalStateException.class, () -> { + transaction.prepareDeletions(Collections.singletonMap(id, vertex), Collections.emptyMap()); + }); + Mockito.verify(transaction, Mockito.never()).queryEdgesFromBackend(Mockito.any()); + Mockito.verify(transaction, Mockito.never()).doRemove(Mockito.any()); + } + @Test public void testBatchDecisionsRemainFixedAfterOriginChanges() { ConditionQuery root = new ConditionQuery(HugeType.VERTEX); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/AuthTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/AuthTest.java index aa4ac8cce7..58875393db 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/AuthTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/AuthTest.java @@ -28,6 +28,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.hugegraph.HugeException; +import org.apache.hugegraph.HugeFactory; import org.apache.hugegraph.HugeGraph; import org.apache.hugegraph.HugeGraphParams; import org.apache.hugegraph.auth.AuthManager; @@ -45,6 +46,7 @@ import org.apache.hugegraph.backend.cache.Cache; import org.apache.hugegraph.backend.id.Id; import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.exception.NotFoundException; import org.apache.hugegraph.schema.VertexLabel; import org.apache.hugegraph.testutil.Assert; import org.apache.hugegraph.testutil.Whitebox; @@ -1757,6 +1759,76 @@ public void testDelProject() { }); } + @Test + public void testDeleteProjectAfterRequestCleanup() { + Assume.assumeTrue("skip this test for hstore", + !Objects.equals("hstore", System.getProperty("backend"))); + Assume.assumeTrue("skip this test for null", System.getProperty("backend") != null); + AuthManager authManager = graph().authManager(); + HugeProject project = makeProject("cleanup_project", ""); + Id projectId = authManager.createProject(project); + HugeProject saved = authManager.getProject(projectId); + Id retainedId = authManager.createProject(makeProject("retained_project", "")); + HugeProject retained = authManager.getProject(retainedId); + Assert.assertEquals(3, authManager.listAccessByTarget(saved.targetId(), -1).size()); + Id userId = authManager.createUser(makeUser("membership_user", "pass")); + authManager.createBelong(makeBelong(userId, saved.adminGroupId())); + Id retainedBelongId = authManager.createBelong(makeBelong(userId, retained.adminGroupId())); + + // Match FINISHED cleanup between REST creation and deletion requests. + HugeFactory.closeCurrentThreadTransactions(); + authManager.deleteProject(projectId); + HugeFactory.closeCurrentThreadTransactions(); + + Assert.assertEquals(0, authManager.listAccessByGroup(saved.adminGroupId(), -1).size()); + Assert.assertEquals(0, authManager.listAccessByGroup(saved.opGroupId(), -1).size()); + Assert.assertEquals(0, authManager.listAccessByTarget(saved.targetId(), -1).size()); + Assert.assertEquals(3, authManager.listAccessByTarget(retained.targetId(), -1).size()); + Assert.assertEquals(0, authManager.listBelongByGroup(saved.adminGroupId(), -1).size()); + Assert.assertEquals(1, authManager.listBelongByUser(userId, -1).size()); + Assert.assertEquals(retainedBelongId, authManager.getBelong(retainedBelongId).id()); + Assert.assertEquals(userId, authManager.getUser(userId).id()); + Assert.assertEquals(projectId, authManager.createProject(makeProject("cleanup_project", ""))); + Assert.assertEquals(0, authManager.listBelongByGroup(saved.adminGroupId(), -1).size()); + } + + @Test + public void testDeleteUserWithRelationsAfterRequestCleanup() { + Assume.assumeTrue("skip this test for hstore", + !Objects.equals("hstore", System.getProperty("backend"))); + Assume.assumeTrue("skip this test for null", System.getProperty("backend") != null); + AuthManager authManager = graph().authManager(); + Id deletedUser = authManager.createUser(makeUser("deleted_member", "pass")); + Id retainedUser = authManager.createUser(makeUser("retained_member", "pass")); + Id group = authManager.createGroup(makeGroup("shared_group")); + authManager.createBelong(makeBelong(deletedUser, group)); + Id retainedBelong = authManager.createBelong(makeBelong(retainedUser, group)); + + HugeFactory.closeCurrentThreadTransactions(); + authManager.deleteUser(deletedUser); + HugeFactory.closeCurrentThreadTransactions(); + + Assert.assertThrows(NotFoundException.class, () -> authManager.getUser(deletedUser)); + Assert.assertEquals(0, authManager.listBelongByUser(deletedUser, -1).size()); + Assert.assertEquals(1, authManager.listBelongByGroup(group, -1).size()); + Assert.assertEquals(retainedBelong, authManager.getBelong(retainedBelong).id()); + Assert.assertEquals(retainedUser, authManager.getUser(retainedUser).id()); + } + + @Test + public void testDeleteUserWithoutRelationsAfterRequestCleanup() { + Assume.assumeTrue("skip this test for hstore", + !Objects.equals("hstore", System.getProperty("backend"))); + Assume.assumeTrue("skip this test for null", System.getProperty("backend") != null); + AuthManager authManager = graph().authManager(); + Id userId = authManager.createUser(makeUser("isolated_user", "pass")); + HugeFactory.closeCurrentThreadTransactions(); + authManager.deleteUser(userId); + HugeFactory.closeCurrentThreadTransactions(); + Assert.assertThrows(NotFoundException.class, () -> authManager.getUser(userId)); + Assert.assertEquals(0, authManager.listBelongByUser(userId, -1).size()); + } + @Test public void testUpdateProject() { Assume.assumeTrue("skip this test for hstore", diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java index f18d656c08..73900b8606 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java @@ -23,6 +23,7 @@ import org.apache.hugegraph.masterelection.GlobalMasterInfo; import org.apache.hugegraph.meta.MetaManager; import org.apache.hugegraph.meta.PdMetaDriver; +import org.apache.hugegraph.task.StandardTaskSchedulerTxTest; import org.apache.hugegraph.task.TaskAndResultSchedulerTest; import org.apache.hugegraph.testutil.Utils; import org.apache.hugegraph.util.Log; @@ -48,6 +49,7 @@ RestoreCoreTest.class, TaskCoreTest.class, TaskAndResultSchedulerTest.class, + StandardTaskSchedulerTxTest.class, AuthTest.class, MultiGraphsTest.class, RamTableTest.class diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java index 24a905427c..6bcfafd161 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/IndexLabelCoreTest.java @@ -22,6 +22,7 @@ import org.apache.hugegraph.HugeException; import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.HugeGraphParams; import org.apache.hugegraph.exception.ExistedException; import org.apache.hugegraph.exception.NoIndexException; import org.apache.hugegraph.exception.NotFoundException; @@ -31,10 +32,12 @@ import org.apache.hugegraph.schema.Userdata; import org.apache.hugegraph.schema.VertexLabel; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Whitebox; import org.apache.hugegraph.type.HugeType; import org.apache.hugegraph.type.define.IndexType; import org.apache.hugegraph.type.define.WriteType; import org.apache.hugegraph.util.DateUtil; +import org.apache.hugegraph.util.Events; import org.apache.tinkerpop.gremlin.process.traversal.P; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.T; @@ -1297,6 +1300,81 @@ public void testRemoveIndexLabelOfVertex() { }); } + @Test + public void testUpdateCachedVertexAfterIndexRemoval() throws Exception { + super.initPropertyKeys(); + SchemaManager schema = graph().schema(); + schema.vertexLabel("person").properties("name", "city") + .primaryKeys("name").create(); + schema.indexLabel("personByCity").onV("person").secondary() + .by("city").create(); + Vertex original = graph().addVertex(T.label, "person", "name", "cache-test", "city", "old"); + graph().tx().commit(); + Vertex cached = graph().vertices(original.id()).next(); + graph().tx().commit(); + // Schema cache eviction must not make a retained vertex's index list authoritative. + HugeGraphParams params = Whitebox.getInternalState(graph(), "params"); + params.schemaEventHub().notify(Events.CACHE, "clear", null).get(); + + schema.indexLabel("personByCity").remove(); + Assert.assertThrows(NotFoundException.class, () -> schema.getIndexLabel("personByCity")); + cached.property("city", "new"); + graph().tx().commit(); + Assert.assertEquals("new", graph().vertices(original.id()).next().value("city")); + graph().tx().commit(); + schema.indexLabel("personByCity").onV("person").secondary().by("city").create(); + schema.propertyKey("extra").asText().create(); + schema.vertexLabel("person").properties("extra").nullableKeys("extra").append(); + schema.indexLabel("personByExtra").onV("person").secondary().by("extra").create(); + graph().taskScheduler().waitUntilAllTasksCompleted(30); + cached.property("city", "newer"); + graph().tx().commit(); + Assert.assertFalse(graph().traversal().V().has("city", "new").hasNext()); + Assert.assertEquals(original.id(), graph().traversal().V().has("city", "newer").next().id()); + graph().tx().commit(); + cached.remove(); + graph().tx().commit(); + Assert.assertFalse(graph().vertices(original.id()).hasNext()); + } + + @Test + public void testUpdateCachedEdgeAfterIndexRemoval() throws Exception { + super.initPropertyKeys(); + SchemaManager schema = graph().schema(); + schema.vertexLabel("author").properties("id", "name").primaryKeys("id").create(); + schema.vertexLabel("book").properties("name").primaryKeys("name").create(); + schema.edgeLabel("authored").singleTime().link("author", "book") + .properties("contribution").create(); + schema.indexLabel("authoredByContri").onE("authored").secondary() + .by("contribution").create(); + Vertex author = graph().addVertex(T.label, "author", "id", 1, "name", "author"); + Vertex book = graph().addVertex(T.label, "book", "name", "book"); + Edge original = author.addEdge("authored", book, "contribution", "old"); + graph().tx().commit(); + Edge cached = graph().edges(original.id()).next(); + graph().tx().commit(); + HugeGraphParams params = Whitebox.getInternalState(graph(), "params"); + params.schemaEventHub().notify(Events.CACHE, "clear", null).get(); + + schema.indexLabel("authoredByContri").remove(); + Assert.assertThrows(NotFoundException.class, () -> schema.getIndexLabel("authoredByContri")); + cached.property("contribution", "new"); + graph().tx().commit(); + Assert.assertEquals("new", graph().edges(original.id()).next().value("contribution")); + graph().tx().commit(); + schema.indexLabel("authoredByContri").onE("authored").secondary() + .by("contribution").create(); + graph().taskScheduler().waitUntilAllTasksCompleted(30); + cached.property("contribution", "newer"); + graph().tx().commit(); + Assert.assertFalse(graph().traversal().E().has("contribution", "new").hasNext()); + Assert.assertEquals(original.id(), graph().traversal().E().has("contribution", "newer").next().id()); + graph().tx().commit(); + cached.remove(); + graph().tx().commit(); + Assert.assertFalse(graph().edges(original.id()).hasNext()); + } + @Test public void testRemoveIndexLabelOfEdge() { super.initPropertyKeys(); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/MultiGraphsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/MultiGraphsTest.java index bb45b47e3b..91cdcea426 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/MultiGraphsTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/MultiGraphsTest.java @@ -413,8 +413,9 @@ public void testCreateGraphsWithMultiDisksForRocksDB() { }, e -> { Throwable root = HugeException.rootCause(e); Assert.assertInstanceOf(RocksDBException.class, root); - Assert.assertContains("While mkdir if missing", + Assert.assertContains("Cannot lock database for open/recovery", root.getMessage()); + Assert.assertContains("/g", root.getMessage()); }); destroyGraphs(ImmutableList.of(g1)); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index 335130ede5..a50b66c756 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -5480,10 +5480,35 @@ public void testQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin() { "confirmType", 2, "type", 1, "kid", 2); graph.addVertex(T.label, "test", "name", "诚信文明", "confirmType", 3, "type", 1, "kid", 3); + graph.addVertex(T.label, "test", "name", "诚信", + "confirmType", 4, "type", 1, "kid", 4); this.assertQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin(); this.commitTx(); this.assertQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin(); + + // Every confirmType branch and the search index reach the threshold. + // Only type=0 is selective, independently of joint-index iteration order. + int kid = 5; + for (int confirmType : new int[]{0, 1, 3}) { + graph.addVertex(T.label, "test", "name", "诚信", "confirmType", confirmType, + "type", 1, "kid", kid++); + } + this.commitTx(); + Object tx = Whitebox.invoke(graph.getClass(), "graphTransaction", graph); + Object threshold = Whitebox.getInternalState(tx, "indexTx.indexIntersectThresh"); + try { + Whitebox.setInternalState(tx, "indexTx.indexIntersectThresh", 2); + List vertices = graph.traversal().V() + .has("type", 0) + .has("confirmType", P.within(0, 1, 3)) + .has("name", Text.contains("诚信")) + .toList(); + Assert.assertEquals(1, vertices.size()); + assertContains(vertices, T.label, "test", "kid", 0); + } finally { + Whitebox.setInternalState(tx, "indexTx.indexIntersectThresh", threshold); + } } private void assertQueryByJointIndexesWithSearchAndTwoRangeIndexesAndWithin() { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/task/StandardTaskSchedulerTxTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/task/StandardTaskSchedulerTxTest.java new file mode 100644 index 0000000000..e4ede91138 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/task/StandardTaskSchedulerTxTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.task; + +import java.util.Collections; + +import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.core.BaseCoreTest; +import org.junit.Assert; +import org.junit.Test; + +public class StandardTaskSchedulerTxTest extends BaseCoreTest { + + @Test + public void testTaskQueryDoesNotOpenUpperGraphTransaction() { + HugeGraph graph = this.graph(); + TaskScheduler scheduler = graph.taskScheduler(); + try { + // The task worker must also be quiescent after graph startup. + Assert.assertFalse(scheduler.call(() -> graph.tx().isOpen())); + + scheduler.tasks(TaskStatus.NEW, 1L, null).hasNext(); + Assert.assertFalse(scheduler.call(() -> graph.tx().isOpen())); + + scheduler.tasks(Collections.singletonList(IdGenerator.of(9999999L))) + .hasNext(); + Assert.assertFalse(scheduler.call(() -> graph.tx().isOpen())); + } finally { + scheduler.call(() -> { + if (graph.tx().isOpen()) { + graph.tx().rollback(); + } + return null; + }); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java index e081423b8e..fc8fc8a3f4 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/testutil/Utils.java @@ -38,6 +38,10 @@ public class Utils { public static final String CONF_PATH = "hugegraph.properties"; public static HugeGraph open() { + return HugeFactory.open(getConf()); + } + + private static String configPath() { String confPath = System.getProperty("config_path"); if (confPath == null || confPath.isEmpty()) { confPath = CONF_PATH; @@ -49,7 +53,7 @@ public static HugeGraph open() { // ignored Exception } - return HugeFactory.open(getLocalConfig(confPath)); + return confPath; } private static PropertiesConfiguration getLocalConfig(String path) { @@ -97,21 +101,7 @@ public static Date date(String rawDate) { } public static PropertiesConfiguration getConf() { - String confFile = Utils.class.getClassLoader() - .getResource(CONF_PATH).getPath(); - File file = new File(confFile); - E.checkArgument(file.exists() && file.isFile() && file.canRead(), - "Need to specify a readable config file rather than:" + - " %s", file.toString()); - - PropertiesConfiguration config; - try { - config = new Configurations().properties(file); - } catch (ConfigurationException e) { - throw new HugeException("Unable to load config file: %s", - e, confFile); - } - return config; + return getLocalConfig(configPath()); } public static void println(String message) { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 4189c1692d..eafa9aa3b6 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -17,8 +17,11 @@ package org.apache.hugegraph.unit; +import org.apache.hugegraph.backend.store.rocksdb.RocksDBSnapshotRestoreTest; +import org.apache.hugegraph.HugeFactoryTest; import org.apache.hugegraph.api.auth.GraphSpaceAuthPayloadTest; import org.apache.hugegraph.api.auth.GraphSpaceGroupAPITest; +import org.apache.hugegraph.auth.ContextTaskTest; import org.apache.hugegraph.auth.StandardAuthManagerV2Test; import org.apache.hugegraph.auth.WsAndHttpBasicAuthHandlerTest; import org.apache.hugegraph.backend.page.QueryListTest; @@ -44,6 +47,7 @@ import org.apache.hugegraph.unit.cache.RamTableTest; import org.apache.hugegraph.unit.cmd.InitStoreConfigTest; import org.apache.hugegraph.unit.core.AnalyzerTest; +import org.apache.hugegraph.unit.core.BackendSessionPoolTest; import org.apache.hugegraph.unit.core.BackendMutationTest; import org.apache.hugegraph.unit.core.BackendStoreInfoTest; import org.apache.hugegraph.unit.core.ConditionQueryFlattenTest; @@ -60,6 +64,7 @@ import org.apache.hugegraph.unit.core.IdHolderTest; import org.apache.hugegraph.unit.core.LocksTableTest; import org.apache.hugegraph.unit.core.PageStateTest; +import org.apache.hugegraph.unit.core.ConfigPathTest; import org.apache.hugegraph.unit.core.QueryResultsTest; import org.apache.hugegraph.unit.core.QueryTest; import org.apache.hugegraph.unit.core.RangeTest; @@ -91,6 +96,7 @@ import org.apache.hugegraph.unit.serializer.TextBackendEntryTest; import org.apache.hugegraph.unit.serializer.TextSerializerTest; import org.apache.hugegraph.unit.store.RamIntObjectMapTest; +import org.apache.hugegraph.unit.traversal.OltpTraverserTest; import org.apache.hugegraph.unit.traversal.ShortestPathTraverserTest; import org.apache.hugegraph.unit.util.CompressUtilTest; import org.apache.hugegraph.unit.util.JsonUtilTest; @@ -120,6 +126,8 @@ GraphSpaceGroupAPITest.class, GraphSpaceAuthPayloadTest.class, StandardAuthManagerV2Test.class, + ContextTaskTest.class, + HugeFactoryTest.class, AuthMetaManagerTest.class, /* api space */ @@ -156,12 +164,14 @@ RowLockTest.class, AnalyzerTest.class, BackendMutationTest.class, + BackendSessionPoolTest.class, ConditionTest.class, StandardHugeGraphClearBackendTest.class, ConditionQueryFlattenTest.class, GraphIndexTransactionTest.class, GraphTransactionTest.class, QueryTest.class, + ConfigPathTest.class, QueryResultsTest.class, QueryListTest.class, RangeTest.class, @@ -182,6 +192,7 @@ TaskSchedulerServerInfoTest.class, HugeGraphAuthProxyTest.class, SchemaElementTest.class, + OltpTraverserTest.class, ShortestPathTraverserTest.class, /* cmd */ @@ -200,6 +211,7 @@ /* rocksdb */ RocksDBSessionsTest.class, + RocksDBSnapshotRestoreTest.class, RocksDBSessionTest.class, RocksDBCountersTest.class, RocksDBTableQueryByIdsTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendSessionPoolTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendSessionPoolTest.java new file mode 100644 index 0000000000..d84d782c34 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendSessionPoolTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +import org.apache.hugegraph.backend.store.BackendSession; +import org.apache.hugegraph.backend.store.BackendSessionPool; +import org.apache.hugegraph.unit.FakeObjects; +import org.junit.Assert; +import org.junit.Test; + +public class BackendSessionPoolTest { + + private static final long TIMEOUT_SECONDS = 10L; + private static final long CLOSE_HOLD_SECONDS = 30L; + + @Test + public void testConcurrentAcquireDoesNotLeaveSessionOnClosedBackend() + throws Exception { + TestSessionPool pool = new TestSessionPool(); + pool.open(); + + AtomicReference closeFailure = new AtomicReference<>(); + CountDownLatch acquireAttempted = new CountDownLatch(1); + FutureTask acquisition = + new FutureTask<>(() -> { + acquireAttempted.countDown(); + return pool.getOrNewSession(); + }); + + Thread closer = new Thread(() -> { + try { + pool.getOrNewSession(); + pool.close(); + } catch (Throwable e) { + closeFailure.set(e); + } + }, "backend-session-closer"); + Thread acquirer = new Thread(acquisition, "backend-session-acquirer"); + + closer.start(); + try { + Assert.assertTrue("last-session close did not enter doClose", + pool.closeEntered.await(TIMEOUT_SECONDS, + TimeUnit.SECONDS)); + + acquirer.start(); + Assert.assertTrue("acquirer did not start", + acquireAttempted.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + boolean acquiredWhileClosePaused = + awaitBorrowerInCloseWindow(pool, closer, acquirer); + BackendSession activeSession = acquiredWhileClosePaused ? + acquisition.get(TIMEOUT_SECONDS, + TimeUnit.SECONDS) : null; + pool.allowClose.countDown(); + + closer.join(TimeUnit.SECONDS.toMillis(TIMEOUT_SECONDS)); + acquirer.join(TimeUnit.SECONDS.toMillis(TIMEOUT_SECONDS)); + Assert.assertFalse("closer did not finish", closer.isAlive()); + Assert.assertFalse("acquirer did not finish", acquirer.isAlive()); + Assert.assertNull(closeFailure.get()); + if (acquiredWhileClosePaused) { + Assert.assertNotNull(activeSession); + Assert.assertTrue("new session should still be open", + activeSession.opened()); + Assert.assertFalse("pool should still count the new active session", + pool.closed()); + + Assert.assertTrue("doClose closed the backend while a newly " + + "acquired session remained active", + pool.opened()); + } else { + try { + acquisition.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assert.fail("late acquire should be rejected after backend close"); + } catch (ExecutionException e) { + Assert.assertTrue("late acquire must fail because backend is closed", + e.getCause() instanceof BackendClosedException); + Assert.assertFalse("backend should be closed before rejecting late acquire", + pool.opened()); + Assert.assertTrue("pool should count no active session after rejection", + pool.closed()); + } + } + } finally { + pool.allowClose.countDown(); + closer.join(TimeUnit.SECONDS.toMillis(TIMEOUT_SECONDS)); + if (acquirer.getState() != Thread.State.NEW) { + acquirer.join(TimeUnit.SECONDS.toMillis(TIMEOUT_SECONDS)); + } + } + } + + private static boolean awaitBorrowerInCloseWindow(TestSessionPool pool, + Thread closer, + Thread acquirer) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (System.nanoTime() < deadline) { + if (pool.acquirerEnteredNewSession.getCount() == 0L) { + return true; + } + ThreadInfo info = ManagementFactory.getThreadMXBean() + .getThreadInfo(acquirer.getId()); + if (info != null && info.getThreadState() == Thread.State.BLOCKED && + info.getLockOwnerId() == closer.getId() && + info.getLockInfo() != null && + info.getLockInfo().getClassName().equals(TestSessionPool.class.getName())) { + return false; + } + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1L)); + } + throw new AssertionError("acquirer neither entered newSession nor blocked on close"); + } + + private static final class TestSessionPool extends BackendSessionPool { + + private final AtomicBoolean opened; + private final AtomicReference session; + private final CountDownLatch closeEntered; + private final CountDownLatch allowClose; + private final CountDownLatch acquirerEnteredNewSession; + + private TestSessionPool() { + super(FakeObjects.newConfig(), "test"); + this.opened = new AtomicBoolean(); + this.session = new AtomicReference<>(); + this.closeEntered = new CountDownLatch(1); + this.allowClose = new CountDownLatch(1); + this.acquirerEnteredNewSession = new CountDownLatch(1); + } + + @Override + public void open() { + this.opened.set(true); + } + + @Override + protected boolean opened() { + return this.opened.get(); + } + + @Override + public BackendSession session() { + return this.session.get(); + } + + @Override + protected BackendSession newSession() { + if (Thread.currentThread().getName().equals("backend-session-acquirer")) { + this.acquirerEnteredNewSession.countDown(); + } + if (!this.opened.get()) { + throw new BackendClosedException(); + } + BackendSession newSession = new TestSession(); + this.session.set(newSession); + return newSession; + } + + @Override + protected void doClose() { + this.closeEntered.countDown(); + try { + if (!this.allowClose.await(CLOSE_HOLD_SECONDS, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to finish doClose"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("interrupted while waiting to finish doClose", e); + } + this.opened.set(false); + } + } + + private static final class TestSession extends BackendSession.AbstractBackendSession { + + @Override + public void open() { + this.opened = true; + } + + @Override + public void close() { + this.opened = false; + } + + @Override + public Object commit() { + return null; + } + + @Override + public void rollback() { + // No transaction state in this test session. + } + + @Override + public boolean hasChanges() { + return false; + } + } + + private static final class BackendClosedException extends RuntimeException { + + private static final long serialVersionUID = 1L; + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConfigPathTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConfigPathTest.java new file mode 100644 index 0000000000..87b482ac89 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/ConfigPathTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.hugegraph.testutil.Utils; +import org.junit.Assert; +import org.junit.Test; + +public class ConfigPathTest { + + @Test + public void testExplicitConfigAlsoAppliesToGraphCopies() throws Exception { + String previous = System.getProperty("config_path"); + Path config = Files.createTempFile("hugegraph-config-path-", ".properties"); + try { + Files.write(config, "hbase.port=22181\nstore=isolated\n" + .getBytes(StandardCharsets.UTF_8)); + System.setProperty("config_path", config.toString()); + Assert.assertEquals(22181, Utils.getConf().getInt("hbase.port")); + Assert.assertEquals("isolated", Utils.getConf().getString("store")); + Utils.getConf().setProperty("store", "copy"); + Assert.assertEquals("isolated", Utils.getConf().getString("store")); + } finally { + if (previous == null) { + System.clearProperty("config_path"); + } else { + System.setProperty("config_path", previous); + } + Files.deleteIfExists(config); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java index 5dbc96c5c6..f54c2083c0 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/rocksdb/RocksDBSessionsTest.java @@ -19,15 +19,24 @@ import java.io.File; import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.io.FileUtils; import org.apache.hugegraph.backend.store.rocksdb.RocksDBMetrics; import org.apache.hugegraph.backend.store.rocksdb.RocksDBOptions; import org.apache.hugegraph.backend.store.rocksdb.RocksDBSessions; import org.apache.hugegraph.backend.store.rocksdb.RocksDBStdSessions; +import org.apache.hugegraph.backend.store.rocksdb.RocksDBStore; +import org.apache.hugegraph.backend.store.rocksdb.RocksDBStoreProvider; import org.apache.hugegraph.backend.store.rocksdbsst.RocksDBSstSessions; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Whitebox; import org.apache.hugegraph.unit.FakeObjects; import org.junit.Test; import org.rocksdb.RocksDBException; @@ -37,6 +46,122 @@ public class RocksDBSessionsTest extends BaseRocksDBUnitTest { + @Test + public void testAdapterToplingTruncateWithMultipleKeys() throws Exception { + this.assertAdapterTruncate(true); + } + + @Test + public void testAdapterStandardTruncateWithMultipleKeys() throws Exception { + this.assertAdapterTruncate(false); + } + + private void assertAdapterTruncate(boolean topling) throws Exception { + List tables = ImmutableList.of(TABLE, "single", "empty", "multiple"); + this.rocks.createTable("single", "empty", "multiple"); + // Deliberately unordered, including empty and unsigned byte boundaries. + byte[][] keys = {new byte[]{(byte) 0xff}, new byte[]{0}, new byte[]{}, + new byte[]{(byte) 0x80}, new byte[]{0, (byte) 0xff}, new byte[]{0x7f}}; + for (String table : ImmutableList.of(TABLE, "multiple")) { + for (byte[] key : keys) { + this.rocks.session().put(table, key, new byte[]{42}); + } + } + this.rocks.session().put("single", new byte[]{(byte) 0xff}, new byte[]{43}); + this.commit(); + Object opened = Whitebox.getInternalState(this.rocks, "rocksdb"); + Map handles = Whitebox.getInternalState(opened, "cfHandles"); + Map before = new HashMap<>(handles); + RocksDBStore store = new RocksDBStore.RocksDBGraphStore(new RocksDBStoreProvider(), "db", "store") { + @Override + protected List tableNames() { + return tables; + } + }; + Whitebox.setInternalState(store, "sessions", this.rocks); + Map databases = Whitebox.getInternalState(store, "dbs"); + databases.put(DB_PATH, this.rocks); + // Select the actual adapter branch, independently of the loaded JNI. + // This fixture also runs unchanged with the externally loaded TP JNI on Linux. + Whitebox.setInternalState(store, "toplingProvider", topling); + store.truncate(); + for (String table : tables) { + Assert.assertTrue(this.rocks.existsTable(table)); + Assert.assertNull(this.rocks.session().keyRange(table)); + for (byte[] key : keys) { + Assert.assertNull(this.rocks.session().get(table, key)); + } + if (topling) { + Assert.assertSame(before.get(table), handles.get(table)); + } else { + Assert.assertNotSame(before.get(table), handles.get(table)); + } + } + for (String table : tables) { + this.rocks.session().put(table, new byte[]{(byte) 0xff}, new byte[]{44}); + } + this.commit(); + for (String table : tables) { + Assert.assertArrayEquals(new byte[]{44}, this.rocks.session().get(table, new byte[]{(byte) 0xff})); + } + // A second truncate covers the single-key path for every formerly empty CF. + store.truncate(); + for (String table : tables) { + Assert.assertNull(this.rocks.session().keyRange(table)); + } + } + + @Test + public void testClearTablesWithoutRecreatingColumnFamilies() + throws RocksDBException, InterruptedException { + final String table2 = "test-table2"; + this.rocks.createTable(table2); + this.rocks.session().put(TABLE, getBytes("person:1"), + getBytes("James")); + this.rocks.session().put(table2, getBytes("person:2"), + getBytes("Tom")); + this.commit(); + + AtomicReference failure = new AtomicReference<>(); + Thread thread = new Thread(() -> { + try { + this.rocks.clearTables(ImmutableList.of(TABLE, table2)); + } catch (Throwable e) { + failure.set(e); + } + }); + thread.start(); + thread.join(); + + Assert.assertNull(failure.get()); + Assert.assertNull(this.rocks.session().get(TABLE, + getBytes("person:1"))); + Assert.assertNull(this.rocks.session().get(table2, + getBytes("person:2"))); + Assert.assertTrue(this.rocks.existsTable(TABLE)); + Assert.assertTrue(this.rocks.existsTable(table2)); + AtomicInteger sessionCount = + Whitebox.getInternalState(this.rocks, "sessionCount"); + Assert.assertEquals(1, sessionCount.get()); + } + + @Test + public void testDatabaseOpenedDoesNotCreateSession() throws RocksDBException { + HugeConfig config = FakeObjects.newConfig(); + String path = DB_PATH + "/opened"; + RocksDBSessions sessions = + new RocksDBStdSessions(config, "db", "store", path, path); + AtomicInteger sessionCount = + Whitebox.getInternalState(sessions, "sessionCount"); + + Assert.assertEquals(0, sessionCount.get()); + Assert.assertTrue(sessions.databaseOpened()); + Assert.assertEquals(0, sessionCount.get()); + + sessions.forceCloseRocksDB(); + Assert.assertFalse(sessions.databaseOpened()); + } + @Test public void testTable() throws RocksDBException { final String TABLE2 = "test-table2"; @@ -135,6 +260,278 @@ public void testSnapshot() throws RocksDBException, IOException { } } + @Test + public void testSnapshotWithSeparateWalDirectory() throws Exception { + String dataPath = DB_PATH + "/separate-data"; + String walPath = DB_PATH + "/separate-wal"; + String snapshotPath = SNAPSHOT_PATH + "/separate-rocks"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(snapshotPath)); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + try { + sessions.createTable(TABLE); + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James")); + sessions.session().put(TABLE, getBytes("person:2gname"), + getBytes("Lisa")); + sessions.session().commit(); + + sessions.createSnapshot(snapshotPath); + + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James2")); + sessions.session().put(TABLE, getBytes("person:3gname"), + getBytes("After")); + sessions.session().commit(); + Assert.assertEquals("James2", getString(sessions.session().get( + TABLE, getBytes("person:1gname")))); + + sessions.resumeSnapshot(snapshotPath); + + Assert.assertEquals("James", getString(sessions.session().get( + TABLE, getBytes("person:1gname")))); + Assert.assertEquals("Lisa", getString(sessions.session().get( + TABLE, getBytes("person:2gname")))); + Assert.assertNull(sessions.session().get(TABLE, + getBytes("person:3gname"))); + Assert.assertEquals(0, readWalLogs(dataPath).size()); + assertNoResumeResidue(walPath); + } finally { + sessions.close(); + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath + ".resume-aside")); + File snapshotFile = FileUtils.getFile(SNAPSHOT_PATH); + if (snapshotFile.exists()) { + FileUtils.forceDelete(snapshotFile); + } + } + } + + + @Test + public void testResumeWhenDataDirectoryIsInsideWal() throws Exception { + String walPath = nestedRoot("parent-wal"); + String dataPath = walPath + "/nested-data"; + resumeNestedAndExpectSnapshot(dataPath, walPath); + } + + @Test + public void testResumeWhenWalDirectoryIsInsideData() throws Exception { + String dataPath = nestedRoot("parent-data"); + String walPath = dataPath + "/nested-wal"; + resumeNestedAndExpectSnapshot(dataPath, walPath); + } + + + private static String nestedRoot(String name) { + return System.getProperty("java.io.tmpdir") + "/nested-wal-closure/" + name; + } + + private void resumeNestedAndExpectSnapshot(String dataPath, String walPath) + throws Exception { + String snapshotPath = SNAPSHOT_PATH + "/nested-rocks"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath).getParentFile()); + FileUtils.forceMkdir(FileUtils.getFile(dataPath).getParentFile()); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + try { + sessions.createTable(TABLE); + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James")); + sessions.session().commit(); + sessions.createSnapshot(snapshotPath); + sessions.session().put(TABLE, getBytes("person:1gname"), + getBytes("James2")); + sessions.session().commit(); + sessions.resumeSnapshot(snapshotPath); + Assert.assertEquals("James", getString(sessions.session().get( + TABLE, getBytes("person:1gname")))); + } finally { + sessions.close(); + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + File snapshotFile = FileUtils.getFile(SNAPSHOT_PATH); + if (snapshotFile.exists()) { + FileUtils.forceDelete(snapshotFile); + } + } + } + + @Test + public void testSeparateWalInstallsCheckpointTail() throws Exception { + String dataPath = DB_PATH + "/tail-data"; + String walPath = DB_PATH + "/tail-wal"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath + ".resume-aside")); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + boolean open = true; + try { + sessions.createTable(TABLE); + sessions.close(); + open = false; + byte[] checkpoint = "checkpoint-tail".getBytes("UTF-8"); + byte[] live = "live-longer-tail".getBytes("UTF-8"); + FileUtils.forceMkdir(FileUtils.getFile(dataPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath)); + FileUtils.writeByteArrayToFile(new File(dataPath, "000123.log"), + checkpoint); + FileUtils.writeByteArrayToFile(new File(walPath, "000123.log"), live); + FileUtils.writeByteArrayToFile(new File(walPath, "000124.log"), + "later".getBytes("UTF-8")); + + Whitebox.invoke(RocksDBStdSessions.class, + "replaceSeparateWalDirectory", sessions); + + assertSameWalLogs(Collections.singletonMap("000123.log", checkpoint), + readWalLogs(walPath)); + Assert.assertFalse(new File(dataPath, "000123.log").exists()); + Assert.assertFalse(new File(walPath, "000124.log").exists()); + assertNoResumeResidue(walPath); + } finally { + if (open) { + sessions.close(); + } + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.deleteDirectory(FileUtils.getFile(walPath + ".resume-aside")); + } + } + + + @Test + public void testDataInsideWalKeepsDataAndInstallsTail() throws Exception { + String walPath = nestedRoot("contain-wal"); + String dataPath = walPath + "/data"; + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + FileUtils.forceMkdir(FileUtils.getFile(dataPath).getParentFile()); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + boolean open = true; + try { + sessions.createTable(TABLE); + sessions.close(); + open = false; + byte[] checkpoint = "checkpoint-tail".getBytes("UTF-8"); + FileUtils.forceMkdir(FileUtils.getFile(dataPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath)); + FileUtils.writeByteArrayToFile(new File(dataPath, "000123.log"), + checkpoint); + FileUtils.writeByteArrayToFile(new File(walPath, "000123.log"), + "live-longer-tail".getBytes("UTF-8")); + FileUtils.writeByteArrayToFile(new File(walPath, "000124.log"), + "later".getBytes("UTF-8")); + File marker = new File(dataPath, "keep-data.txt"); + FileUtils.writeByteArrayToFile(marker, "kept".getBytes("UTF-8")); + + Whitebox.invoke(RocksDBStdSessions.class, + "replaceSeparateWalDirectory", sessions); + + assertSameWalLogs(Collections.singletonMap("000123.log", checkpoint), + readWalLogs(walPath)); + Assert.assertTrue(marker.isFile()); + Assert.assertFalse(new File(dataPath, "000123.log").exists()); + assertNoResumeResidue(walPath); + } finally { + if (open) { + sessions.close(); + } + FileUtils.deleteDirectory(FileUtils.getFile(walPath)); + } + } + + @Test + public void testWalInsideDataInstallsTail() throws Exception { + String dataPath = nestedRoot("inside-data"); + String walPath = dataPath + "/wal"; + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + FileUtils.forceMkdir(FileUtils.getFile(walPath)); + HugeConfig config = FakeObjects.newConfig(); + RocksDBSessions sessions = new RocksDBStdSessions(config, "db", "store", + dataPath, walPath); + boolean open = true; + try { + sessions.createTable(TABLE); + sessions.close(); + open = false; + byte[] checkpoint = "checkpoint-tail".getBytes("UTF-8"); + FileUtils.writeByteArrayToFile(new File(dataPath, "000123.log"), + checkpoint); + FileUtils.writeByteArrayToFile(new File(walPath, "000124.log"), + "later".getBytes("UTF-8")); + + Whitebox.invoke(RocksDBStdSessions.class, + "replaceSeparateWalDirectory", sessions); + + assertSameWalLogs(Collections.singletonMap("000123.log", checkpoint), + readWalLogs(walPath)); + Assert.assertFalse(new File(dataPath, "000123.log").exists()); + Assert.assertFalse(new File(walPath, "000124.log").exists()); + assertNoResumeResidue(walPath); + } finally { + if (open) { + sessions.close(); + } + FileUtils.deleteDirectory(FileUtils.getFile(dataPath)); + } + } + + private static Map readWalLogs(String directory) throws IOException { + Map logs = new HashMap<>(); + File dir = FileUtils.getFile(directory); + File[] children = dir.listFiles(); + if (children == null) { + return logs; + } + for (File child : children) { + String name = child.getName(); + int dot = name.lastIndexOf('.'); + if (!child.isFile() || dot <= 0 || !".log".equals(name.substring(dot))) { + continue; + } + boolean digits = true; + for (int i = 0; i < dot; i++) { + if (!Character.isDigit(name.charAt(i))) { + digits = false; + break; + } + } + if (digits) { + logs.put(name, FileUtils.readFileToByteArray(child)); + } + } + return logs; + } + + private static void assertSameWalLogs(Map expected, + Map actual) { + Assert.assertEquals(expected.keySet(), actual.keySet()); + for (Map.Entry entry : expected.entrySet()) { + Assert.assertArrayEquals(entry.getValue(), actual.get(entry.getKey())); + } + } + + private static void assertNoResumeResidue(String walPath) { + File wal = FileUtils.getFile(walPath); + File[] children = wal.getParentFile().listFiles(); + Assert.assertNotNull(children); + for (File child : children) { + String name = child.getName(); + Assert.assertFalse(name.startsWith(wal.getName() + ".resume-aside-")); + Assert.assertFalse(name.startsWith(wal.getName() + ".resume-staging-")); + } + } + @Test public void testCopySessions() throws RocksDBException { Assert.assertFalse(this.rocks.closed()); @@ -190,7 +587,8 @@ public void testIngestSst() throws RocksDBException { Assert.assertFalse(sstSessions.existsTable(TABLE1)); Assert.assertFalse(sstSessions.existsTable(TABLE2)); - RocksDBSessions rocks = new RocksDBStdSessions(config, "db", "store", sstPath, sstPath); + RocksDBSessions rocks = + new RocksDBStdSessions(config, "db", "store", sstPath, sstPath); // Will ingest sst file of TABLE1 rocks.createTable(TABLE1); Assert.assertEquals(ImmutableList.of("1000"), diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/traversal/OltpTraverserTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/traversal/OltpTraverserTest.java new file mode 100644 index 0000000000..ba4683b469 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/traversal/OltpTraverserTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.traversal; + +import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.config.CoreOptions; +import org.apache.hugegraph.traversal.algorithm.OltpTraverser; +import org.apache.hugegraph.type.define.CollectionType; +import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.tinkerpop.gremlin.structure.Transaction; +import org.junit.After; +import org.junit.Test; +import org.mockito.Mockito; + +public class OltpTraverserTest extends BaseUnitTest { + + @After + public void teardown() { + OltpTraverser.destroy(); + } + + @Test + public void testCloseOpenTransaction() { + HugeGraph graph = graph(); + Transaction tx = graph.tx(); + Mockito.when(tx.isOpen()).thenReturn(true); + + new TestTraverser(graph).close(); + + Mockito.verify(tx).close(); + } + + @Test + public void testCloseClosedTransaction() { + HugeGraph graph = graph(); + Transaction tx = graph.tx(); + Mockito.when(tx.isOpen()).thenReturn(false); + + new TestTraverser(graph).close(); + + Mockito.verify(tx, Mockito.never()).close(); + } + + private static HugeGraph graph() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + Transaction tx = Mockito.mock(Transaction.class); + Mockito.when(graph.tx()).thenReturn(tx); + Mockito.when(graph.option(CoreOptions.OLTP_COLLECTION_TYPE)) + .thenReturn(CollectionType.JCF); + Mockito.when(graph.option(CoreOptions.OLTP_CONCURRENT_THREADS)) + .thenReturn(1); + return graph; + } + + private static class TestTraverser extends OltpTraverser { + + TestTraverser(HugeGraph graph) { + super(graph); + } + } +} diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 988b472b09..91ddbaad21 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -33,9 +33,9 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz -# 2nd stage: runtime env +# Shared runtime environment # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:11-jre-jammy +FROM eclipse-temurin:11-jre-noble AS runtime LABEL maintainer="HugeGraph Docker Maintainers " @@ -56,21 +56,82 @@ RUN apt-get -q update \ dumb-init \ procps \ curl \ + util-linux \ vim \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-*/ /hugegraph-store/ - # 2. Init docker script COPY hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh EXPOSE 8520 -VOLUME /hugegraph-store HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8520/v1/health >/dev/null ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["./docker-entrypoint.sh"] + +# Topling runtime packages. The current native library requires glibc 2.38+. +FROM runtime AS topling-runtime + +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + libaio1t64 \ + libbz2-1.0 \ + libjemalloc2 \ + liblz4-1 \ + libsnappy1v5 \ + liburing2 \ + zlib1g \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Build the formal component-local Topling distribution on Linux amd64. +FROM topling-runtime AS build-topling + +COPY --from=build /pkg/ /pkg/ +WORKDIR /pkg +RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends --no-install-suggests \ + rsync \ + unzip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && STORE_DIR=$(find /pkg/hugegraph-store -maxdepth 1 -type d \ + -name 'apache-hugegraph-store-*' ! -name '*-topling' \ + -print -quit) \ + && test -n "$STORE_DIR" \ + && VERSION=${STORE_DIR##*/apache-hugegraph-store-} \ + && install-dist/scripts/build-topling-distribution.sh store "$VERSION" + +# Topling image: contains both providers and selects ToplingDB by default. +# Build with: docker build --platform linux/amd64 --target topling ... +FROM topling-runtime AS topling + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="topling" +COPY --from=build-topling /pkg/hugegraph-store/apache-hugegraph-store-*-topling/ /hugegraph-store/ +ENV HG_STORE_ROCKSDB_PROVIDER="topling" \ + HG_STORE_ENFORCE_PROVIDER_MARKER="true" +RUN mkdir -p /hugegraph-store/storage /hugegraph-store/topling-storage +VOLUME ["/hugegraph-store/storage", "/hugegraph-store/topling-storage"] + +# Keep the standard image as the final/default target for existing builds. +FROM runtime AS standard + +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +ARG SOURCE_REVISION=local +LABEL org.opencontainers.image.source="${SOURCE_REPOSITORY}" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" + +LABEL org.apache.hugegraph.rocksdb-runtime="standard" +COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-*/ /hugegraph-store/ +ENV HG_STORE_ROCKSDB_PROVIDER="rocksdb" +# Preserve the historical root volume for anonymous-volume upgrades. +VOLUME /hugegraph-store diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index f12df4427e..fbb5d8b651 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -386,6 +386,20 @@ For Docker and Kubernetes deployment details, see [Deployment Guide](docs/deploy --- +## Stopping a Store node + +For an unpacked distribution, run `bin/stop-hugegraph-store.sh`. It waits up to +30 seconds for process exit. A timeout returns a nonzero status and retains +`bin/pid` for diagnosis; it does not force-kill the process or remove its data. +Check the Store log and thread dump before taking further action. + +Spring owns shutdown: new RPCs are refused, active RPCs are cancelled, and context +close waits for their callbacks and scan/TTL workers to release resources. The +Store engine then stops and joins partition Raft services before releasing +databases. A stuck callback keeps shutdown pending rather than allowing its +database to close underneath it. Do not add a separate JVM hook that closes +those databases concurrently. + ## Documentation Comprehensive documentation for HugeGraph Store: diff --git a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh index 8ea9022a33..73a2a3b03a 100755 --- a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh +++ b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh @@ -53,11 +53,33 @@ require_env "HG_STORE_RAFT_ADDRESS" # ── Defaults ────────────────────────────────────────────────────────── : "${HG_STORE_GRPC_PORT:=8500}" : "${HG_STORE_REST_PORT:=8520}" -: "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +: "${HG_STORE_ROCKSDB_PROVIDER:=rocksdb}" + +case "${HG_STORE_ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) + log "ERROR: HG_STORE_ROCKSDB_PROVIDER must be rocksdb or topling" + exit 2 + ;; +esac +if [[ -z "${HG_STORE_DATA_PATH:-}" ]]; then + if [[ "${HG_STORE_ROCKSDB_PROVIDER}" == "topling" ]]; then + HG_STORE_DATA_PATH="$(pwd)/topling-storage" + else + HG_STORE_DATA_PATH="$(pwd)/storage" + fi +fi +export TOPLINGDB_ROCKSDB_PROVIDER="${HG_STORE_ROCKSDB_PROVIDER}" +./bin/verify-rocksdb-provider.sh \ + store \ + "${HG_STORE_ROCKSDB_PROVIDER}" \ + "${HG_STORE_DATA_PATH}" \ + "${HG_STORE_ENFORCE_PROVIDER_MARKER:-false}" # ── Build SPRING_APPLICATION_JSON ───────────────────────────────────── SPRING_APPLICATION_JSON="$(cat < + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/common-topling.sh + + bin + common-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/prepare-topling.sh + + bin + prepare-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/preload-topling.sh + + bin + preload-topling.sh + 0755 + + + + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/verify-rocksdb-provider.sh + + bin + verify-rocksdb-provider.sh + 0755 + + + diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index 88165e47b6..c17138ea4c 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -64,6 +64,13 @@ else echo "Unsupported architecture: $arch" fi +# preload rocksdb/toplingdb +if [ ! -r "$BIN/preload-topling.sh" ]; then + echo "Required RocksDB runtime selector not found: $BIN/preload-topling.sh" >&2 + exit 1 +fi +source "$BIN/preload-topling.sh" || exit 1 + ##pd/store max user processes, ulimit -u # Reduce the maximum number of processes that can be opened by a normal dev/user export PROC_LIMITN=1024 @@ -136,7 +143,7 @@ fi # check jdk version JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}') -if [[ $? -ne 0 || $JAVA_VERSION < $EXPECT_JDK_VERSION ]]; then +if [[ $? -ne 0 || $JAVA_VERSION -lt $EXPECT_JDK_VERSION ]]; then echo "Please make sure that the JDK is installed and the version >= $EXPECT_JDK_VERSION" >> ${OUTPUT} exit 1 fi @@ -225,18 +232,28 @@ if [ "$(ps -ef | grep -v grep | grep java | grep -cE "${CONF}")" -ne 0 ]; then fi echo "Starting HG-StoreServer..." +BOOT_JARS=("${LIB}"/hg-store-node-*.jar) +if [ "${#BOOT_JARS[@]}" -ne 1 ] || [ ! -f "${BOOT_JARS[0]}" ]; then + echo "Expected exactly one HugeGraph Store executable JAR in ${LIB}" >> "${OUTPUT}" + exit 1 +fi +BOOT_JAR="${BOOT_JARS[0]}" +JAVA_MAIN=(-jar "$BOOT_JAR") +if [ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]; then + JAVA_MAIN=(-cp "${TOPLING_RUNTIME_CLASSPATH}:${BOOT_JAR}" \ + org.springframework.boot.loader.JarLauncher) +fi # Turn on security check if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphStoreServer in daemon mode..." if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar & + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" & else - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar >> ${OUTPUT} 2>&1 & + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 & fi PID="$!" # Write pid to file @@ -248,12 +265,11 @@ else echo "$$" > "$PID_FILE" echo "[+pid] $$" if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" else - exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} -jar \ - -Dspring.config.location=${CONF}/application.yml \ - ${LIB}/hg-store-node-*.jar >> ${OUTPUT} 2>&1 + exec ${JAVA} -Dname="HugeGraphStore" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + -Dspring.config.location=${CONF}/application.yml "${JAVA_MAIN[@]}" \ + >> ${OUTPUT} 2>&1 fi fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh index 8f898df7da..e9f503b02a 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/stop-hugegraph-store.sh @@ -41,8 +41,8 @@ if [ ! -f ${PID_FILE} ]; then fi PID=`cat $PID_FILE` -kill_process_and_wait "HugeGraphStoreServer" "$PID" "$SERVER_SHUTDOWN_TIMEOUT_S" - -if [ $? -eq 0 ]; then +if kill_process_and_wait "HugeGraphStoreServer" "$PID" "$SERVER_SHUTDOWN_TIMEOUT_S"; then rm "$PID_FILE" +else + exit 1 fi diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh index e45249a04b..579cc12c49 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/util.sh @@ -434,3 +434,21 @@ function kill_process_and_wait() { kill_process "$process_name" "$pid" wait_for_shutdown "$process_name" "$pid" "$timeout_s" } + +# Find HugeGraph server directory in parent path using prefix glob. +# Usage: find_hugegraph_server_dir "/path/to/parent" +# Returns: first matching directory path or empty string +function find_hugegraph_server_dir() { + local parent_dir="$1" + if [ -z "$parent_dir" ]; then + parent_dir="$(cd "${TOP:-$(pwd)}"/.. && pwd)" + fi + local found="" + for d in "$parent_dir"/apache-hugegraph-server*; do + if [ -d "$d" ]; then + found="$d" + break + fi + done + echo "$found" +} diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 0315c4b4fe..2d667afdb1 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -32,3 +32,6 @@ rocksdb: write_buffer_size: 32000000 # For each rocksdb, the number of memtables reaches this value for writing to disk. min_write_buffer_number_to_merge: 16 + # To enable ToplingDB: set provider to 'topling' (requires the shared ToplingDB runtime) + # ToplingDB config is loaded from conf/rocksdb_store.yaml + # provider: topling diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml new file mode 100644 index 0000000000..e60d866953 --- /dev/null +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml @@ -0,0 +1,164 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# common parameters +http: + # normally parent path of db path + document_root: ./library/rocksdb_resource + listening_ports: '127.0.0.1:2013' + auto_start_http: false +setenv: + StrSimpleEnvNameNotOverwrite: StringValue + IntSimpleEnvNameNotOverwrite: 16384 + OverwriteThisEnv: + #comment: overwrite is default to false + overwrite: true + value: force overwrite this env by overwrite true +Cache: + lru_cache: + class: LRUCache + params: + capacity: 8G + num_shard_bits: -1 + strict_capacity_limit: false + high_pri_pool_ratio: 0.5 + use_adaptive_mutex: false + metadata_charge_policy: kFullChargeCacheMetadata +Statistics: + stat: + class: default + params: + discard_tickers: + - rocksdb.block.cache + - rocksdb.block.cachecompressed + - rocksdb.block + - rocksdb.memtable.payload.bytes.at.flush + - rocksdb.memtable.garbage.bytes.at.flush + - rocksdb.txn + - rocksdb.blobdb + - rocksdb.row.cache + - rocksdb.number.block + - rocksdb.bloom.filter + - rocksdb.persistent + - rocksdb.sim.block.cache + discard_histograms: + # comment: .... + - rocksdb.blobdb + - rocksdb.bytes.compressed + - rocksdb.bytes.decompressed + - rocksdb.num.index.and.filter.blocks.read.per.level + - rocksdb.num.data.blocks.read.per.level + - rocksdb.compression.times.nanos + - rocksdb.decompression.times.nanos + - rocksdb.read.block.get.micros + - rocksdb.write.raw.block.micros + # comment end of array + #stats_level: kAll + stats_level: kDisableAll +MemTableRepFactory: + cspp: + class: cspp + params: + mem_cap: 16G + use_vm: false + token_use_idle: true + chunk_size: 16K + convert_to_sst: kFileMmap + sync_sst_file: false + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + cspp_memtab_sst: + class: CSPPMemTabTable + params: # empty params + bb: + class: BlockBasedTable + params: + checksum: kCRC32c + block_size: 4K + block_restart_interval: 16 + index_block_restart_interval: 1 + metadata_block_size: 4K + enable_index_compression: true + block_cache: "${lru_cache}" + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + block_cache_compressed: + persistent_cache: + filter_policy: + dispatch: + class: DispatcherTable + params: + default: bb + readers: + BlockBasedTable: bb + CSPPMemTabTable: cspp_memtab_sst + level_writers: [ bb, bb, bb, bb, bb, bb ] +CFOptions: + default: + max_write_buffer_number: 6 + memtable_factory: "${cspp}" + write_buffer_size: 128M + # set target_file_size_base as small as 512K is to make many SST files, + # thus key prefix cache can present efficiency + target_file_size_base: 64M + target_file_size_multiplier: 1 + table_factory: dispatch + max_bytes_for_level_base: 512M + max_bytes_for_level_multiplier: 10 + level_compaction_dynamic_level_bytes: false + level0_slowdown_writes_trigger: 20 + level0_stop_writes_trigger: 36 + level0_file_num_compaction_trigger: 2 + merge_operator: uint64add # support merge + level_compaction_dynamic_file_size: true + optimize_filters_for_hits: true + allow_merge_memtables: true + min_write_buffer_number_to_merge: 2 + compression_per_level: + - kNoCompression + - kNoCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression + - kSnappyCompression +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false # this is important, must be false to hugegraph + max_background_compactions: -1 + max_subcompactions: 4 + max_level1_subcompactions: 0 + inplace_update_support: false + WAL_size_limit_MB: 0 + statistics: "${stat}" + max_manifest_file_size: 100M + max_background_jobs: 8 + # Java org.rocksdb.WriteBatch has no Topling mmap WAL. With + # memtable_as_log_index enabled, every batch write fails: + # "WriteBatch has no mmap wal". Keep the option false for this client. + # convert_to_sst is still Topling-specific. Do not open data written by + # this profile with the standard RocksDB provider; restore a snapshot + # taken before migration instead. + compaction_readahead_size: 0 + memtable_as_log_index: false diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java index c793ed96f2..4027659c7f 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java @@ -17,7 +17,6 @@ package org.apache.hugegraph.store.node; -import org.apache.hugegraph.store.node.listener.ContextClosedListener; import org.apache.hugegraph.store.node.listener.PdConfigureListener; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -31,9 +30,6 @@ @SpringBootApplication public class StoreNodeApplication { - //TODO Is this OK? - private final AppShutdownHook shutdownHook = new AppShutdownHook(Thread.currentThread()); - public static void main(String[] args) { start(); } @@ -55,9 +51,7 @@ public static void start() { } SpringApplication application = new SpringApplication(StoreNodeApplication.class); PdConfigureListener listener = new PdConfigureListener(); - ContextClosedListener closedListener = new ContextClosedListener(); application.addListeners(listener); - application.addListeners(closedListener); ConfigurableApplicationContext context = application.run(); listener.setContext(context); System.out.println("StoreNodeApplication started."); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcShutdownBarrier.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcShutdownBarrier.java new file mode 100644 index 0000000000..a9a6a9f95a --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/GrpcShutdownBarrier.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.grpc; + +import java.util.HashSet; +import java.util.Set; + +import org.lognet.springboot.grpc.GRpcGlobalInterceptor; +import org.springframework.stereotype.Component; + +import io.grpc.ForwardingServerCallListener; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import lombok.extern.slf4j.Slf4j; + +/** Waits for terminal application callbacks, not just transport termination. */ +@Slf4j +@Component +@GRpcGlobalInterceptor +public class GrpcShutdownBarrier implements ServerInterceptor { + + private final Set calls = new HashSet<>(); + private boolean closing; + + public synchronized void stopAcceptingCalls() { + this.closing = true; + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, Metadata headers, + ServerCallHandler next) { + Object token = new Object(); + synchronized (this) { + if (this.closing) { + call.close(Status.UNAVAILABLE.withDescription("Store is stopping"), new Metadata()); + return new ServerCall.Listener() { }; + } + this.calls.add(token); + } + try { + return new ForwardingServerCallListener.SimpleForwardingServerCallListener( + next.startCall(call, headers)) { + @Override + public void onCancel() { + try { + super.onCancel(); + } finally { + finished(token); + } + } + + @Override + public void onComplete() { + try { + super.onComplete(); + } finally { + finished(token); + } + } + }; + } catch (RuntimeException | Error e) { + finished(token); + throw e; + } + } + + private synchronized void finished(Object token) { + this.calls.remove(token); + this.notifyAll(); + } + + public synchronized void awaitCallbacks() { + boolean interrupted = false; + try { + while (!this.calls.isEmpty()) { + try { + this.wait(5000); + if (!this.calls.isEmpty()) { + log.warn("Waiting for {} RPC callbacks before closing databases", this.calls.size()); + } + } catch (InterruptedException e) { + interrupted = true; + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java index 7d01fa3db4..f920cccb45 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreStreamImpl.java @@ -17,7 +17,9 @@ package org.apache.hugegraph.store.node.grpc; +import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; import org.apache.hugegraph.store.grpc.state.ScanState; @@ -31,6 +33,8 @@ import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; +import io.grpc.Context; +import io.grpc.Status; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -47,6 +51,53 @@ public class HgStoreStreamImpl extends HgStoreStreamGrpc.HgStoreStreamImplBase { private AppConfig appConfig; private HgStoreWrapperEx wrapper; private ThreadPoolExecutor executor; + private boolean closing; + private final Set scans = ConcurrentHashMap.newKeySet(); + + /** Close admission before cancelling the gRPC server. */ + public synchronized void stopAcceptingScans() { + this.closing = true; + } + + /** Cancel streams, then drain queued cleanup tasks. Never discard the queue. */ + public void shutdownScans() { + stopAcceptingScans(); + for (Runnable cancel : this.scans.toArray(new Runnable[0])) { + cancel.run(); + } + ThreadPoolExecutor current = getRealExecutor(); + if (current != null) { + current.shutdown(); + } + } + + private synchronized void checkAcceptingScans() { + if (this.closing) { + throw Status.UNAVAILABLE.withDescription("Store scans are stopping") + .asRuntimeException(); + } + } + + private StreamObserver register(StreamObserver observer) { + Context context = Context.current(); + Runnable cancel = new Runnable() { + @Override + public synchronized void run() { + if (scans.contains(this)) { + try { + observer.onError(Status.CANCELLED.asRuntimeException()); + } catch (RuntimeException e) { + log.warn("Failed to cancel scan", e); + } finally { + scans.remove(this); + } + } + } + }; + this.scans.add(cancel); + context.addListener(ignored -> cancel.run(), Runnable::run); + return observer; + } private HgStoreWrapperEx getWrapper() { if (this.wrapper == null) { @@ -60,20 +111,16 @@ private HgStoreWrapperEx getWrapper() { return this.wrapper; } - public ThreadPoolExecutor getRealExecutor() { - return executor; + public synchronized ThreadPoolExecutor getRealExecutor() { + return this.executor; } - public ThreadPoolExecutor getExecutor() { + public synchronized ThreadPoolExecutor getExecutor() { + checkAcceptingScans(); if (this.executor == null) { - synchronized (this) { - if (this.executor == null) { - AppConfig.ThreadPoolScan scan = this.appConfig.getThreadPoolScan(); - this.executor = - HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), scan.getMax(), - scan.getQueue()); - } - } + AppConfig.ThreadPoolScan scan = this.appConfig.getThreadPoolScan(); + this.executor = HgExecutorUtil.createExecutor("hg-scan", scan.getCore(), + scan.getMax(), scan.getQueue()); } return this.executor; } @@ -95,27 +142,32 @@ public ScanState getState() { } @Override - public StreamObserver scan(StreamObserver response) { - return ScanStreamResponse.of(response, getWrapper(), getExecutor(), appConfig); + public synchronized StreamObserver scan(StreamObserver response) { + checkAcceptingScans(); + return register(ScanStreamResponse.of(response, getWrapper(), getExecutor(), appConfig)); } @Override public void scanOneShot(ScanStreamReq request, StreamObserver response) { + checkAcceptingScans(); ScanOneShotResponse.scanOneShot(request, response, getWrapper()); } @Override - public StreamObserver scanBatch(StreamObserver response) { - return ScanBatchResponse3.of(response, getWrapper(), getExecutor()); + public synchronized StreamObserver scanBatch(StreamObserver response) { + checkAcceptingScans(); + return register(ScanBatchResponse3.of(response, getWrapper(), getExecutor())); } @Override - public StreamObserver scanBatch2(StreamObserver response) { - return ScanBatchResponseFactory.of(response, getWrapper(), getExecutor()); + public synchronized StreamObserver scanBatch2(StreamObserver response) { + checkAcceptingScans(); + return register(ScanBatchResponseFactory.of(response, getWrapper(), getExecutor())); } @Override public void scanBatchOneShot(ScanStreamBatchReq request, StreamObserver response) { + checkAcceptingScans(); ScanBatchOneShotResponse.scanOneShot(request, response, getWrapper()); } } diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java index 56ce1f45f3..b164837a82 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ParallelScanIterator.java @@ -88,7 +88,12 @@ private ParallelScanIterator(Supplier> iter this.maxInQueue = maxWorkThreads * 2; // Edge sorted requires a larger queue queue = new LinkedBlockingQueue<>(maxInQueue * 2); - createScanner(); + try { + createScanner(); + } catch (RuntimeException e) { + close(); + throw e; + } } public static ParallelScanIterator of( @@ -144,18 +149,21 @@ public List next() { return t; } + void requestStop() { + this.finished = true; + } + @Override public void close() { - finished = true; + requestStop(); + List pending; synchronized (scanners) { - scanners.forEach(scanner -> { - scanner.close(); - }); + pending = new ArrayList<>(scanners); } + // Never hold a registry lock while waiting for a scanner's iterator lock. + pending.forEach(KVScanner::close); synchronized (pauseScanners) { - pauseScanners.forEach(s -> { - s.close(); - }); + pauseScanners.clear(); } queue.clear(); } @@ -179,10 +187,15 @@ private void createScanner() { */ private void wakeUpScanner() { synchronized (pauseScanners) { - if (!pauseScanners.isEmpty()) { + if (!finished && !pauseScanners.isEmpty()) { KVScanner scanner = pauseScanners.poll(); if (scanner != null) { - executor.execute(() -> scanner.scanKV()); + try { + executor.execute(() -> scanner.scanKV()); + } catch (java.util.concurrent.RejectedExecutionException e) { + scanner.close(); + throw e; + } } } } @@ -195,7 +208,11 @@ private void wakeUpScanner() { */ private void suspendScanner(KVScanner scanner) { synchronized (pauseScanners) { - pauseScanners.add(scanner); + if (!finished) { + pauseScanners.add(scanner); + } else { + scanner.close(); + } } } @@ -218,30 +235,43 @@ private void quitScanner(KVScanner scanner) { */ private boolean putData(List data) { try { - this.queue.put(data); + while (!finished && !this.queue.offer(data, 100, TimeUnit.MILLISECONDS)) { + // Cancellation must release a producer whose client stopped consuming. + } } catch (InterruptedException e) { log.error("exception ", e); this.finished = true; return false; } - return this.queue.size() < maxInQueue; + return !finished && this.queue.size() < maxInQueue; } private boolean putData(List data, boolean hasNext) { + boolean locked = false; try { - queueLock.lock(); - this.queue.put(data); + while (!finished && !(locked = queueLock.tryLock(100, TimeUnit.MILLISECONDS))) { + // An ordered producer may be waiting behind a cancelled scanner. + } + if (!locked) { + return false; + } + while (!finished && !this.queue.offer(data, 100, TimeUnit.MILLISECONDS)) { + // Recheck cancellation while the output queue is full. + } } catch (InterruptedException e) { - log.error("exception ", e); + Thread.currentThread().interrupt(); this.finished = true; return false; } finally { - if (!hasNext) { + if (locked && finished) { + while (queueLock.isHeldByCurrentThread()) { + queueLock.unlock(); + } + } else if (locked && !hasNext) { queueLock.unlock(); } } - // Data not ended, thread continues to execute - return hasNext || this.queue.size() < maxInQueue; + return !finished && (hasNext || this.queue.size() < maxInQueue); } private synchronized KVPair getIterator() { @@ -328,12 +358,12 @@ public void scanKV() { iteratorLock.lock(); try { long entriesSize = 0, bodySize = 0; - while (canNext && !closed) { + while (canNext && !closed && !finished) { iterator = this.getIterator(); if (iterator == null) { break; } - while (iterator.hasNext() && entriesSize < batchSize && + while (!closed && !finished && iterator.hasNext() && entriesSize < batchSize && bodySize < maxBodySize && counter < limit && !closed) { KV kv = KV.of(iterator.next()); @@ -366,8 +396,14 @@ public void scanKV() { } catch (Exception e) { log.error("exception {}", e); } finally { + // putData(..., true) intentionally keeps this lock across batches to + // serialize ordered output. The scanner must always release it before + // suspending or quitting, including when the next iterator is empty. + while (queueLock.isHeldByCurrentThread()) { + queueLock.unlock(); + } iteratorLock.unlock(); - if (iterator != null && counter < limit && !closed) { + if (iterator != null && counter < limit && !closed && !finished) { suspendScanner(this); } else { quitScanner(this); @@ -381,6 +417,7 @@ public void close() { try { if (iterator != null) { iterator.close(); + iterator = null; } } finally { iteratorLock.unlock(); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java index 1234a5b3de..2caa1ecd56 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchOneShotResponse.java @@ -30,6 +30,7 @@ import com.google.protobuf.ByteString; +import io.grpc.Context; import io.grpc.Status; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -54,6 +55,9 @@ public static void scanOneShot(ScanStreamBatchReq request, String graph = request.getHeader().getGraph(); ScanQueryRequest queryRequest = request.getQueryRequest(); + if (Context.current().isCancelled()) { + return; + } ScanIterator iterator = getIterator(graph, queryRequest, wrapper); KvPageRes.Builder resBuilder = KvPageRes.newBuilder(); @@ -70,7 +74,8 @@ public static void scanOneShot(ScanStreamBatchReq request, int count = 0; try { - while (iterator.hasNext()) { + while (!Context.current().isCancelled() && + !Thread.currentThread().isInterrupted() && iterator.hasNext()) { if (++count > limit) { break; @@ -86,6 +91,9 @@ public static void scanOneShot(ScanStreamBatchReq request, } + if (Context.current().isCancelled()) { + return; + } responseObserver.onNext(resBuilder.build()); responseObserver.onCompleted(); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java index f4485e1527..c9971e0948 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse.java @@ -20,7 +20,9 @@ import static org.apache.hugegraph.store.node.grpc.ScanUtil.getParallelIterator; import java.util.List; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -59,7 +61,7 @@ public class ScanBatchResponse implements StreamObserver { private final Object stateLock = new Object(); private final Lock iteratorLock = new ReentrantLock(); // Currently traversing iterator - private ScanIterator iterator; + private volatile ScanIterator iterator; // Next send sequence number private volatile int seqNo; // Client consumed sequence number @@ -72,6 +74,7 @@ public class ScanBatchResponse implements StreamObserver { // Last read data time private long activeTime; private volatile State state; + private final AtomicBoolean cancelled = new AtomicBoolean(); public ScanBatchResponse(StreamObserver response, HgStoreWrapperEx wrapper, ThreadPoolExecutor executor) { @@ -92,25 +95,30 @@ public ScanBatchResponse(StreamObserver response, HgStoreWrapperEx wra */ @Override public void onNext(ScanStreamBatchReq request) { + if (this.cancelled.get()) { + return; + } switch (request.getQueryCase()) { case QUERY_REQUEST: // query conditions - executor.execute(() -> { + submit(() -> { startQuery(request.getHeader().getGraph(), request.getQueryRequest()); }); break; case RECEIPT_REQUEST: // Message asynchronous response this.clientSeqNo = request.getReceiptRequest().getTimes(); if (seqNo - clientSeqNo < maxInFlightCount) { + boolean send = false; synchronized (stateLock) { if (state == State.IDLE) { state = State.DOING; - executor.execute(() -> { - sendEntries(); - }); - } else if (state == State.DONE) { - sendNoDataEntries(); + send = true; } } + if (send) { + submit(this::sendEntries); + } else if (state == State.DONE) { + sendNoDataEntries(); + } } break; case CANCEL_REQUEST: // close stream @@ -125,7 +133,7 @@ public void onNext(ScanStreamBatchReq request) { @Override public void onError(Throwable t) { log.error("onError ", t); - closeQuery(); + closeQuery(t); } @Override @@ -138,18 +146,43 @@ public void onCompleted() { * * @param request */ + private void submit(Runnable task) { + if (this.cancelled.get()) { + return; + } + try { + this.executor.execute(task); + } catch (RejectedExecutionException e) { + closeQuery(e); + } + } + private void startQuery(String graphName, ScanQueryRequest request) { - this.query = request; - this.limit = request.getLimit(); - this.count = 0; - this.iterator = getParallelIterator(graphName, request, this.wrapper, executor); - synchronized (stateLock) { - if (state == State.IDLE) { + this.iteratorLock.lock(); + try { + if (this.cancelled.get() || this.iterator != null) { + return; + } + this.query = request; + this.limit = request.getLimit(); + this.count = 0; + this.iterator = getParallelIterator(graphName, request, this.wrapper, executor); + if (this.cancelled.get()) { + closeIter(); + return; + } + synchronized (stateLock) { + if (state != State.IDLE) { + return; + } state = State.DOING; - executor.execute(() -> { - sendEntries(); - }); } + submit(this::sendEntries); + } catch (RuntimeException e) { + closeQuery(e); + log.warn("Failed to start scan", e); + } finally { + this.iteratorLock.unlock(); } } @@ -157,10 +190,25 @@ private void startQuery(String graphName, ScanQueryRequest request) { * Generate iterator */ private void closeQuery() { + closeQuery(null); + } + + private void closeQuery(Throwable error) { + if (!this.cancelled.compareAndSet(false, true)) { + return; + } setStateDone(); + ScanIterator current = this.iterator; + if (current instanceof ParallelScanIterator) { + ((ParallelScanIterator) current).requestStop(); + } try { closeIter(); - this.sender.onCompleted(); + if (error == null) { + this.sender.onCompleted(); + } else { + this.sender.onError(error); + } } catch (Exception e) { log.error("exception ", e); } @@ -169,13 +217,16 @@ private void closeQuery() { } private void closeIter() { + this.iteratorLock.lock(); try { if (this.iterator != null) { this.iterator.close(); this.iterator = null; } } catch (Exception e) { - + log.warn("Failed to close batch scan iterator", e); + } finally { + this.iteratorLock.unlock(); } } @@ -209,7 +260,7 @@ private void sendEntries() { this.sender.onNext(dataBuilder.build()); this.activeTime = System.currentTimeMillis(); } - if (!iterator.hasNext() || this.count >= limit || state == State.DONE) { + if (state == State.DONE || this.count >= limit || !iterator.hasNext()) { closeIter(); this.sender.onNext(KvStream.newBuilder().setOver(true).build()); setStateDone(); @@ -219,14 +270,7 @@ private void sendEntries() { } catch (Throwable e) { if (this.state != State.DONE) { log.error(" send data exception: ", e); - setStateIdle(); - if (this.sender != null) { - try { - this.sender.onError(e); - } catch (Exception ex) { - log.warn("Error when call sender.onError {}", e.getMessage()); - } - } + closeQuery(e); } } finally { iteratorLock.unlock(); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java index d0c52b372e..8710612bde 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanBatchResponse3.java @@ -160,16 +160,19 @@ private static class OrderManager { OrderState state = OrderState.NEW; OrderWorker worker; OrderDeliverer deliverer; + boolean cancelled; synchronized void deal(OrderWorker worker, OrderDeliverer deliverer) { if (log.isDebugEnabled()) { log.debug("Receiving query request."); } - if (this.state == OrderState.NEW) { + if (!this.cancelled && this.state == OrderState.NEW) { this.worker = worker; this.deliverer = deliverer; this.worker.hereWeGo(); this.state = OrderState.WORKING; + } else { + worker.iterator.close(); } } @@ -177,7 +180,9 @@ synchronized void receipt(int receiptTimes) { if (log.isDebugEnabled()) { log.debug("Receiving receipt request."); } - this.worker.setReceipt(receiptTimes); + if (this.worker != null) { + this.worker.setReceipt(receiptTimes); + } } synchronized void finished() { @@ -188,6 +193,7 @@ synchronized void finished() { } synchronized void breakdown() { + this.cancelled = true; if (this.worker != null) { this.worker.breakdown(); } @@ -297,7 +303,12 @@ void hereWeGo() { return; } - executor.execute(() -> working()); + try { + executor.execute(this::working); + } catch (java.util.concurrent.RejectedExecutionException e) { + this.iterator.close(); + throw e; + } Thread.yield(); } @@ -338,7 +349,8 @@ private void working() { Kv.Builder kvBuilder = Kv.newBuilder(); long packageCount = 0; - while (iterator.hasNext()) { + while (!this.breakdown.get() && !Thread.currentThread().isInterrupted() && + iterator.hasNext()) { if (++this.counter > limit) { this.completeFlag.set(true); break; @@ -353,7 +365,7 @@ private void working() { deliverer.deliver(dataBuilder, curTimes.incrementAndGet(), false); Thread.yield(); - if (!this.checkContinue()) { + if (!this.breakdown.get() && !this.checkContinue()) { long start = System.currentTimeMillis(); iterator.wait( HgStoreConst.SCAN_WAIT_CLIENT_TAKING_TIME_OUT_SECONDS * diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java index 5b77a4449a..257b715687 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanOneShotResponse.java @@ -27,6 +27,7 @@ import com.google.protobuf.ByteString; +import io.grpc.Context; import io.grpc.Status; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @@ -57,12 +58,16 @@ public static void scanOneShot(ScanStreamReq request, responseObserver.onError(HgGrpc.toErr("limit<=0, please to invoke stream scan.")); return; } + if (Context.current().isCancelled()) { + return; + } ScanIterator iterator = ScanUtil.getIterator(request, wrapper); int count = 0; try { - while (iterator.hasNext()) { + while (!Context.current().isCancelled() && + !Thread.currentThread().isInterrupted() && iterator.hasNext()) { if (++count > limit) { break; @@ -78,6 +83,9 @@ public static void scanOneShot(ScanStreamReq request, } + if (Context.current().isCancelled()) { + return; + } responseObserver.onNext( resBuilder.setVersion(ScanUtil.responseVersion(request)) .build()); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java index e572862823..7e051fd48d 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/ScanStreamResponse.java @@ -61,6 +61,22 @@ public class ScanStreamResponse implements StreamObserver { private final int waitTime; private final HgChannel channel; private ScanIterator iterator; + private final Object cancellationLock = new Object(); + private Thread worker; + private Thread receiver; + + private void cancel() { + this.isStop.set(true); + this.channel.close(); + synchronized (this.cancellationLock) { + if (this.worker != null) { + this.worker.interrupt(); + } + if (this.receiver != null) { + this.receiver.interrupt(); + } + } + } private long limit = 0; private int times = 0; private long pageSize = 0; @@ -91,6 +107,9 @@ public static ScanStreamResponse of(StreamObserver responseObserver, @Override public void onNext(ScanStreamReq request) { + if (this.isStop.get()) { + return; + } try { this.responseVersion = Math.max( this.responseVersion, ScanUtil.responseVersion(request)); @@ -106,7 +125,7 @@ public void onNext(ScanStreamReq request) { @Override public void onError(Throwable t) { - this.isStop.set(true); + cancel(); this.finishServer(); log.warn("onError from client [ graph: {} , table: {}]; Reason: {}]", graph, table, t.getMessage()); @@ -114,7 +133,7 @@ public void onError(Throwable t) { @Override public void onCompleted() { - this.isStop.set(true); + cancel(); this.finishServer(); } @@ -123,7 +142,6 @@ private void initIterator(ScanStreamReq request) { if (this.isStarted.getAndSet(true)) { return; } - this.iterator = getIterator(request, this.wrapper); this.graph = request.getHeader().getGraph(); this.table = request.getTable(); this.limit = request.getLimit(); @@ -143,7 +161,15 @@ private void initIterator(ScanStreamReq request) { Kv.Builder kvBuilder = Kv.newBuilder(); int pageCount = 0; try { - while (iterator.hasNext()) { + synchronized (this.cancellationLock) { + if (this.isStop.get()) { + return; + } + this.worker = Thread.currentThread(); + } + this.iterator = getIterator(request, this.wrapper); + while (!this.isStop.get() && !Thread.currentThread().isInterrupted() && + iterator.hasNext()) { if (limit > 0 && ++this.total > limit) { break; } @@ -172,10 +198,16 @@ private void initIterator(ScanStreamReq request) { responseObserver.onError(ex); } finally { try { - this.iterator.close(); + if (this.iterator != null) { + this.iterator.close(); + } this.channel.close(); } catch (Exception e) { - + log.warn("Failed to close scan iterator", e); + } finally { + synchronized (this.cancellationLock) { + this.worker = null; + } } } @@ -184,12 +216,7 @@ private void initIterator(ScanStreamReq request) { } catch (Exception e) { StatusRuntimeException ex = HgGrpc.toErr(Status.INTERNAL, null, e); responseObserver.onError(ex); - try { - this.iterator.close(); - this.channel.close(); - } catch (Exception exception) { - - } + this.channel.close(); } /*** Scanning loop end ***/ @@ -205,8 +232,7 @@ private Kv toKv(Kv.Builder kvBuilder, RocksDBSession.BackendColumn col, } private void close() { - this.isStop.set(true); - this.channel.close(); + cancel(); if (!this.finishFlag.get()) { responseObserver.onNext(KvPageRes.newBuilder() .addAllData(Collections.EMPTY_LIST) @@ -225,6 +251,12 @@ private void next(ScanStreamReq request) { KvPageRes.Builder resBuilder; try { + synchronized (this.cancellationLock) { + if (this.isStop.get()) { + return; + } + this.receiver = Thread.currentThread(); + } resBuilder = this.channel.receive(); times++; } catch (Exception e) { @@ -232,6 +264,10 @@ private void next(ScanStreamReq request) { log.error(msg, e); responseObserver.onError(HgGrpc.toErr(msg + e.getMessage())); return; + } finally { + synchronized (this.cancellationLock) { + this.receiver = null; + } } boolean isOver = false; if (resBuilder == null || resBuilder.getDataList() == null || diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java index df8d084e22..f5188b23e3 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/ContextClosedListener.java @@ -17,106 +17,112 @@ package org.apache.hugegraph.store.node.listener; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; -import org.apache.hugegraph.store.HgStoreEngine; +import org.apache.hugegraph.store.node.grpc.GrpcShutdownBarrier; import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; import org.apache.hugegraph.store.node.task.TTLCleaner; +import org.lognet.springboot.grpc.context.GRpcServerInitializedEvent; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import com.alipay.sofa.jraft.Status; -import com.alipay.sofa.jraft.entity.PeerId; - +import io.grpc.Server; import lombok.extern.slf4j.Slf4j; @Slf4j @Service public class ContextClosedListener implements ApplicationListener { + private final List grpcServers = new CopyOnWriteArrayList<>(); + @Autowired HgStoreStreamImpl storeStream; @Autowired TTLCleaner cleaner; + @Autowired + GrpcShutdownBarrier grpcBarrier; + + @EventListener + public void onServerInitialized(GRpcServerInitializedEvent event) { + this.grpcServers.add(event.getServer()); + } @Override public void onApplicationEvent(ContextClosedEvent event) { + // Spring invokes HgStoreNodeService.destroy() after this event. Raft and + // its databases must remain available until local workers have stopped. + this.grpcBarrier.stopAcceptingCalls(); + if (storeStream != null) { + storeStream.stopAcceptingScans(); + } + this.grpcServers.forEach(Server::shutdownNow); + if (cleaner != null) { + // The scheduler can create the worker pool while a job is starting. + stopAndWait(cleaner.getScheduler(), "TTL scheduler"); + stopAndWait(cleaner.getExecutor(), "TTL workers"); + } + if (storeStream != null) { + // Cancelled queued scans must run their finally blocks to release iterators. + storeStream.shutdownScans(); + awaitWorkers(storeStream.getRealExecutor(), "scan workers"); + } + boolean interrupted = false; try { - try { - transferLeaders(); - - synchronized (ContextClosedListener.class) { - ContextClosedListener.class.wait(60 * 1000); - } - - transferLeaders(); - - synchronized (ContextClosedListener.class) { - ContextClosedListener.class.wait(30 * 1000); - } - } catch (Exception e) { - log.info("shutdown hook: ", e); - } - - log.info("closing scan threads...."); - if (storeStream != null) { - ThreadPoolExecutor executor = storeStream.getRealExecutor(); - if (executor != null) { + for (Server server : this.grpcServers) { + while (!server.isTerminated()) { try { - executor.shutdownNow(); - } catch (Exception e) { + if (!server.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("Still waiting for gRPC callbacks before closing databases"); + } + } catch (InterruptedException e) { + interrupted = true; } } } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + this.grpcBarrier.awaitCallbacks(); + log.info("closed gRPC callbacks, scan and TTL workers"); + } - if (cleaner != null) { - ThreadPoolExecutor cleanerExecutor = cleaner.getExecutor(); - if (cleanerExecutor != null) { - try { - cleanerExecutor.shutdownNow(); - } catch (Exception e) { - - } - } - ScheduledExecutorService scheduler = cleaner.getScheduler(); - if (scheduler != null) { - try { - scheduler.shutdownNow(); - } catch (Exception e) { + private static void stopAndWait(ExecutorService executor, String name) { + if (executor == null) { + return; + } + executor.shutdownNow(); + awaitWorkers(executor, name); + } + private static void awaitWorkers(ExecutorService executor, String name) { + if (executor == null) { + return; + } + boolean interrupted = false; + try { + while (!executor.isTerminated()) { + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("Still waiting for {} to stop before closing databases", name); } + } catch (InterruptedException e) { + // An interrupted shutdown thread must not close databases underneath + // a worker that still owns a native iterator. + interrupted = true; } } - } catch (Exception e) { - log.error("ContextClosedListener: ", e); } finally { - log.info("closed scan threads"); - } - } - - private void transferLeaders() { - try { - HgStoreEngine.getInstance().getLeaderPartition() - .forEach(leader -> { - try { - Status status = - leader.getRaftNode().transferLeadershipTo(PeerId.ANY_PEER); - log.info("partition {} transfer leader status: {}", - leader.getGroupId(), status); - } catch (Exception e) { - log.info("partition {} transfer leader error: ", - leader.getGroupId(), e); - } - }); - HgStoreEngine.getInstance().getPartitionEngines().forEach( - ((integer, partitionEngine) -> partitionEngine.getRaftNode() - .shutdown()) - ); - } catch (Exception e) { - log.error("transfer leader failed: " + e.getMessage()); + if (interrupted) { + Thread.currentThread().interrupt(); + } } } } diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java index 94bdc4c6bc..ceecc43399 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/metrics/RocksDBMetricsConst.java @@ -54,12 +54,12 @@ public final class RocksDBMetricsConst { TickerType.BLOCK_CACHE_INDEX_HIT, // Index cache hits. TickerType.BLOCK_CACHE_INDEX_ADD, // Index blocks added to cache. TickerType.BLOCK_CACHE_INDEX_BYTES_INSERT, // Bytes inserted into index cache. - TickerType.BLOCK_CACHE_INDEX_BYTES_EVICT, // Bytes evicted from index cache. + // BLOCK_CACHE_INDEX_BYTES_EVICT removed in RocksDB 8.x TickerType.BLOCK_CACHE_FILTER_MISS, // Filter cache misses. TickerType.BLOCK_CACHE_FILTER_HIT, // Filter cache hits. TickerType.BLOCK_CACHE_FILTER_ADD, // Filter blocks added to cache. TickerType.BLOCK_CACHE_FILTER_BYTES_INSERT, // Bytes inserted in filter cache. - TickerType.BLOCK_CACHE_FILTER_BYTES_EVICT, // Bytes evicted from filter cache. + // BLOCK_CACHE_FILTER_BYTES_EVICT removed in RocksDB 8.x TickerType.BLOCK_CACHE_DATA_MISS, // Data cache misses. TickerType.BLOCK_CACHE_DATA_HIT, // Data cache hits. TickerType.BLOCK_CACHE_DATA_ADD, // Data blocks added to cache. @@ -95,31 +95,31 @@ public final class RocksDBMetricsConst { TickerType.NUMBER_DB_NEXT_FOUND, // Number of successful next operations. TickerType.NUMBER_DB_PREV_FOUND, // Number of successful previous operations. TickerType.ITER_BYTES_READ, // Bytes read by iterators. - TickerType.NO_FILE_CLOSES, // Number of file close operations. + // NO_FILE_CLOSES removed in RocksDB 8.x TickerType.NO_FILE_OPENS, // Number of file open operations. TickerType.NO_FILE_ERRORS, // Number of file errors. TickerType.STALL_MICROS, // Time spent in a stall micro. TickerType.DB_MUTEX_WAIT_MICROS, // Time spent waiting on a mutex. - TickerType.RATE_LIMIT_DELAY_MILLIS, // Rate limiting delay in milliseconds. - TickerType.NO_ITERATORS, // Number of iterators created. + // RATE_LIMIT_DELAY_MILLIS removed in RocksDB 8.x + // NO_ITERATORS removed in RocksDB 8.x TickerType.NUMBER_MULTIGET_BYTES_READ, // Bytes read by multi-get operations. TickerType.NUMBER_MULTIGET_KEYS_READ, // Keys read in multi-get operations. TickerType.NUMBER_MULTIGET_CALLS, // Number of multi-get operations. - TickerType.NUMBER_FILTERED_DELETES, // Number of deletes filtered. + // NUMBER_FILTERED_DELETES removed in RocksDB 8.x TickerType.NUMBER_MERGE_FAILURES, // Number of merge failures. TickerType.BLOOM_FILTER_PREFIX_CHECKED, // Number of prefix bloom filter checks. TickerType.BLOOM_FILTER_PREFIX_USEFUL, // Number of useful prefix bloom filter checks. TickerType.NUMBER_OF_RESEEKS_IN_ITERATION, // Number of reseeks in iteration. TickerType.GET_UPDATES_SINCE_CALLS, // Number of get updates since calls. - TickerType.BLOCK_CACHE_COMPRESSED_MISS, // Misses in compressed block cache. - TickerType.BLOCK_CACHE_COMPRESSED_HIT, // Hits in compressed block cache. - TickerType.BLOCK_CACHE_COMPRESSED_ADD, // Compressed blocks added to cache. - TickerType.BLOCK_CACHE_COMPRESSED_ADD_FAILURES, // Failures adding compressed blocks. + // BLOCK_CACHE_COMPRESSED_MISS removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_HIT removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_ADD removed in RocksDB 8.x + // BLOCK_CACHE_COMPRESSED_ADD_FAILURES removed in RocksDB 8.x TickerType.WAL_FILE_SYNCED, // Number of synced WAL files. TickerType.WAL_FILE_BYTES, // Bytes written to WAL files. TickerType.WRITE_DONE_BY_SELF, // Writes completed by self. TickerType.WRITE_DONE_BY_OTHER, // Writes completed by others. - TickerType.WRITE_TIMEDOUT, // Number of write timeouts. + // WRITE_TIMEDOUT removed in RocksDB 8.x TickerType.WRITE_WITH_WAL, // Writes involving WAL. TickerType.COMPACT_READ_BYTES, // Bytes read during compaction. TickerType.COMPACT_WRITE_BYTES, // Bytes written during compaction. @@ -167,12 +167,10 @@ public final class RocksDBMetricsConst { // Time spent reading blocks during compaction. HistogramType.READ_BLOCK_GET_MICROS, // Time spent reading blocks during get. HistogramType.WRITE_RAW_BLOCK_MICROS, // Time spent writing raw blocks. - HistogramType.STALL_L0_SLOWDOWN_COUNT, // Count of stalls due to L0 slowdown. - HistogramType.STALL_MEMTABLE_COMPACTION_COUNT, - // Count of stalls due to memtable compaction. - HistogramType.STALL_L0_NUM_FILES_COUNT, // Count of stalls due to number of files at L0. - HistogramType.HARD_RATE_LIMIT_DELAY_COUNT, // Count of delays due to hard rate limits. - HistogramType.SOFT_RATE_LIMIT_DELAY_COUNT, // Count of delays due to soft rate limits. + // STALL_L0_SLOWDOWN_COUNT removed in RocksDB 8.x + // Note: The following constants were removed in RocksDB 8.10.2: + // STALL_MEMTABLE_COMPACTION_COUNT, STALL_L0_NUM_FILES_COUNT, + // HARD_RATE_LIMIT_DELAY_COUNT, SOFT_RATE_LIMIT_DELAY_COUNT HistogramType.NUM_FILES_IN_SINGLE_COMPACTION, // Number of files in a single compaction. HistogramType.DB_SEEK, // Latency of database seek operations. HistogramType.WRITE_STALL, // Time spent in write stalls. diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java index 6836fe20b7..ebcdcc45b2 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/task/TTLCleaner.java @@ -299,7 +299,7 @@ private Runnable getTask( LinkedList all = new LinkedList<>(); AtomicBoolean state = new AtomicBoolean(true); AtomicLong partitionCounter = pc.get(id); - while (filter.hasNext() && state.get()) { + while (!Thread.currentThread().isInterrupted() && state.get() && filter.hasNext()) { RocksDBSession.BackendColumn current = filter.next(); byte[] realKey = Arrays.copyOfRange(current.name, 0, current.name.length - Short.BYTES); @@ -313,7 +313,7 @@ private Runnable getTask( all = new LinkedList<>(); } } - if (all.size() > 0 && state.get()) { + if (!Thread.currentThread().isInterrupted() && all.size() > 0 && state.get()) { submitter.submitClean(id, graph, table, all, state, tableCounter, partitionCounter); } diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/config/ToplingProfileConfigTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/config/ToplingProfileConfigTest.java new file mode 100644 index 0000000000..082f2379cd --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/config/ToplingProfileConfigTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.config; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.Test; + +public class ToplingProfileConfigTest { + + @Test + public void testJavaWriteBatchProfileDisablesMmapWalIndex() throws IOException { + assertMemtableAsLogIndexFalse(repoFile( + "hugegraph-store/hg-store-dist/src/assembly/static/conf/rocksdb_store.yaml")); + assertMemtableAsLogIndexFalse(repoFile( + "hugegraph-pd/hg-pd-dist/src/assembly/static/conf/rocksdb_pd.yaml")); + assertMemtableAsLogIndexFalse(repoFile( + "hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml")); + } + + private static void assertMemtableAsLogIndexFalse(Path file) throws IOException { + List lines = Files.readAllLines(file); + int matches = 0; + for (String line : lines) { + String trimmed = line.trim(); + if (trimmed.startsWith("#") || !trimmed.startsWith("memtable_as_log_index:")) { + continue; + } + matches++; + assertEquals(file.toString(), "false", + trimmed.substring("memtable_as_log_index:".length()).trim()); + } + assertTrue(file.toString(), matches == 1); + } + + private static Path repoFile(String relative) { + Path module = Path.of("").toAbsolutePath(); + Path repo = module.getParent().getParent(); + Path file = repo.resolve(relative); + assertTrue(file.toString(), Files.isRegularFile(file)); + return file; + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ContextClosedListenerTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ContextClosedListenerTest.java new file mode 100644 index 0000000000..452e2c7b51 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ContextClosedListenerTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.store.node.grpc.GrpcShutdownBarrier; +import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; +import org.apache.hugegraph.store.node.listener.ContextClosedListener; +import org.apache.hugegraph.store.node.task.TTLCleaner; +import org.junit.Test; +import org.lognet.springboot.grpc.context.GRpcServerInitializedEvent; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import io.grpc.Server; + +public class ContextClosedListenerTest { + + @Test(timeout = 5000) + public void testContextCloseStopsActiveWorkersWithoutPartitionDelay() throws Exception { + ThreadPoolExecutor scan = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ThreadPoolExecutor ttl = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + CountDownLatch started = new CountDownLatch(3); + CountDownLatch interrupted = new CountDownLatch(3); + CountDownLatch cleanupStarted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch cleanupAllowed = new CountDownLatch(1); + Runnable work = () -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + interrupted.countDown(); + cleanupStarted.countDown(); + boolean done = false; + while (!done) { + try { + cleanupAllowed.await(); + done = true; + } catch (InterruptedException ignored) { + // Model resource cleanup which must finish before DB destruction. + } + } + Thread.currentThread().interrupt(); + } + }; + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + try { + HgStoreStreamImpl stream = mock(HgStoreStreamImpl.class); + TTLCleaner cleaner = mock(TTLCleaner.class); + when(stream.getRealExecutor()).thenReturn(scan); + doAnswer(invocation -> { + scan.shutdownNow(); + return null; + }).when(stream).shutdownScans(); + when(cleaner.getExecutor()).thenReturn(ttl); + when(cleaner.getScheduler()).thenReturn(scheduler); + context.getBeanFactory().registerSingleton("storeStream", stream); + context.getBeanFactory().registerSingleton("cleaner", cleaner); + context.register(ContextClosedListener.class, GrpcShutdownBarrier.class); + context.refresh(); + scan.execute(work); + ttl.execute(work); + scheduler.execute(work); + assertTrue("all shutdown participants must be active", started.await(1, TimeUnit.SECONDS)); + + FutureTask closing = new FutureTask<>(() -> { + context.close(); + return null; + }); + Thread closeThread = new Thread(closing, "test-context-close"); + closeThread.setDaemon(true); + closeThread.start(); + // Scheduler shutdown must not let DB destruction overtake resource cleanup. + assertTrue(cleanupStarted.await(1, TimeUnit.SECONDS)); + assertFalse("context close must wait for worker cleanup", closing.isDone()); + cleanupAllowed.countDown(); + closing.get(2, TimeUnit.SECONDS); + assertTrue("close must interrupt active workers", interrupted.await(1, TimeUnit.SECONDS)); + assertTrue(scan.awaitTermination(1, TimeUnit.SECONDS)); + assertTrue(ttl.awaitTermination(1, TimeUnit.SECONDS)); + assertTrue(scheduler.awaitTermination(1, TimeUnit.SECONDS)); + } finally { + cleanupAllowed.countDown(); + release.countDown(); + scan.shutdownNow(); + ttl.shutdownNow(); + scheduler.shutdownNow(); + context.close(); + } + } + + @Test(timeout = 5000) + public void testGrpcCallbacksFinishBeforeBeanDestructionEvenWhenInterrupted() throws Exception { + CountDownLatch awaitingCallbacks = new CountDownLatch(1); + CountDownLatch releaseCallbacks = new CountDownLatch(1); + AtomicBoolean terminated = new AtomicBoolean(); + AtomicBoolean databaseClosed = new AtomicBoolean(); + Server server = mock(Server.class); + when(server.shutdownNow()).thenReturn(server); + when(server.isTerminated()).thenAnswer(invocation -> terminated.get()); + when(server.awaitTermination(5, TimeUnit.SECONDS)).thenAnswer(invocation -> { + awaitingCallbacks.countDown(); + releaseCallbacks.await(); + terminated.set(true); + return true; + }); + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + try { + context.getBeanFactory().registerSingleton("storeStream", mock(HgStoreStreamImpl.class)); + context.getBeanFactory().registerSingleton("cleaner", mock(TTLCleaner.class)); + context.getDefaultListableBeanFactory().registerDisposableBean("database", () -> databaseClosed.set(true)); + context.register(ContextClosedListener.class, GrpcShutdownBarrier.class); + context.refresh(); + context.publishEvent(new GRpcServerInitializedEvent(context, server)); + FutureTask closing = new FutureTask<>(() -> { + context.close(); + return Thread.currentThread().isInterrupted(); + }); + Thread closeThread = new Thread(closing, "test-grpc-context-close"); + closeThread.setDaemon(true); + closeThread.start(); + assertTrue(awaitingCallbacks.await(1, TimeUnit.SECONDS)); + closeThread.interrupt(); + assertFalse("DB must remain open while callbacks own its resources", databaseClosed.get()); + releaseCallbacks.countDown(); + assertTrue("shutdown must preserve interruption", closing.get(2, TimeUnit.SECONDS)); + assertTrue(terminated.get()); + assertTrue(databaseClosed.get()); + } finally { + releaseCallbacks.countDown(); + context.close(); + } + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/GrpcShutdownBarrierTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/GrpcShutdownBarrierTest.java new file mode 100644 index 0000000000..c32bcb0393 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/GrpcShutdownBarrierTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.store.node.grpc.GrpcShutdownBarrier; +import org.junit.Test; + +import io.grpc.CallOptions; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.Server; +import io.grpc.ServerServiceDefinition; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.stub.ClientCalls; +import io.grpc.stub.ServerCalls; +import io.grpc.stub.StreamObserver; + +public class GrpcShutdownBarrierTest { + + @Test(timeout = 7000) + public void testTransportTerminationDoesNotReleaseActiveCallback() throws Exception { + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean finished = new AtomicBoolean(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + MethodDescriptor.Marshaller marshaller = new MethodDescriptor.Marshaller() { + @Override + public InputStream stream(byte[] value) { + return new ByteArrayInputStream(value); + } + @Override + public byte[] parse(InputStream stream) { + return new byte[0]; + } + }; + MethodDescriptor method = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY).setFullMethodName("probe/hold") + .setRequestMarshaller(marshaller).setResponseMarshaller(marshaller).build(); + String name = InProcessServerBuilder.generateName(); + GrpcShutdownBarrier barrier = new GrpcShutdownBarrier(); + Server server = InProcessServerBuilder.forName(name).executor(executor).intercept(barrier) + .addService(ServerServiceDefinition.builder("probe").addMethod(method, + ServerCalls.asyncUnaryCall((request, response) -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + finished.set(true); + } + response.onNext(new byte[0]); + response.onCompleted(); + })).build()).build().start(); + ManagedChannel channel = InProcessChannelBuilder.forName(name).directExecutor().build(); + try { + ClientCalls.asyncUnaryCall(channel.newCall(method, CallOptions.DEFAULT), new byte[0], + new StreamObserver() { + @Override + public void onNext(byte[] value) { + } + @Override + public void onError(Throwable error) { + } + @Override + public void onCompleted() { + } + }); + assertTrue("handler must start", started.await(2, TimeUnit.SECONDS)); + barrier.stopAcceptingCalls(); + server.shutdownNow(); + assertTrue(server.awaitTermination(1, TimeUnit.SECONDS)); + assertFalse("transport termination must not be mistaken for callback exit", finished.get()); + CountDownLatch waiting = new CountDownLatch(1); + FutureTask drained = new FutureTask<>(() -> { + waiting.countDown(); + barrier.awaitCallbacks(); + return null; + }); + Thread waiter = new Thread(drained, "test-rpc-drain"); + waiter.setDaemon(true); + waiter.start(); + assertTrue(waiting.await(1, TimeUnit.SECONDS)); + try { + drained.get(100, TimeUnit.MILLISECONDS); + fail("barrier passed while a handler still owns resources"); + } catch (TimeoutException expected) { + // The active handler must prevent database destruction. + } + release.countDown(); + drained.get(2, TimeUnit.SECONDS); + assertTrue(finished.get()); + } finally { + release.countDown(); + channel.shutdownNow(); + server.shutdownNow(); + executor.shutdown(); + assertTrue("cleanup must finish", executor.awaitTermination(2, TimeUnit.SECONDS)); + } + } +} diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ScanShutdownTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ScanShutdownTest.java new file mode 100644 index 0000000000..ade1cf9913 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/service/ScanShutdownTest.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.service; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.hugegraph.pd.common.KVPair; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.rocksdb.access.ScanIterator; +import org.apache.hugegraph.store.grpc.common.ScanMethod; +import org.apache.hugegraph.store.grpc.common.ScanOrderType; +import org.apache.hugegraph.store.grpc.stream.KvPageRes; +import org.apache.hugegraph.store.grpc.stream.KvStream; +import org.apache.hugegraph.store.grpc.stream.ScanStreamReq; +import org.apache.hugegraph.store.grpc.stream.ScanStreamBatchReq; +import org.apache.hugegraph.store.grpc.stream.ScanQueryRequest; +import org.apache.hugegraph.store.node.AppConfig; +import org.apache.hugegraph.store.node.grpc.HgStoreStreamImpl; +import org.apache.hugegraph.store.node.grpc.HgStoreWrapperEx; +import org.apache.hugegraph.store.node.grpc.ScanBatchResponse; +import org.apache.hugegraph.store.node.grpc.ParallelScanIterator; +import org.apache.hugegraph.store.node.grpc.QueryCondition; +import org.apache.hugegraph.store.node.grpc.ScanOneShotResponse; +import org.apache.hugegraph.store.node.grpc.ScanStreamResponse; +import org.junit.Test; + +import io.grpc.Context; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; + +public class ScanShutdownTest { + + @Test + public void testClosingPreventsLazyExecutorAndStateCreation() { + HgStoreStreamImpl service = new HgStoreStreamImpl(); + service.stopAcceptingScans(); + assertUnavailable(service::getExecutor); + assertUnavailable(service::getState); + assertUnavailable(() -> service.scan(mock(StreamObserver.class))); + service.shutdownScans(); + assertNull(service.getRealExecutor()); + } + + @Test(timeout = 5000) + public void testQueuedStreamCancellationDrainsWithoutOpeningIterator() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch occupied = new CountDownLatch(1); + executor.execute(() -> { + occupied.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + AppConfig config = mock(AppConfig.class); + when(config.getServerWaitTime()).thenReturn(60); + StreamObserver output = mock(StreamObserver.class); + ScanStreamResponse response = ScanStreamResponse.of(output, wrapper, executor, config); + FutureTask request = new FutureTask<>(() -> { + response.onNext(ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL) + .setPageSize(1).setLimit(10).build()); + return null; + }); + Thread caller = new Thread(request, "scan-shutdown-test"); + try { + assertTrue(occupied.await(1, TimeUnit.SECONDS)); + caller.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + while (executor.getQueue().isEmpty() && System.nanoTime() < deadline) { + Thread.yield(); + } + assertEquals(1, executor.getQueue().size()); + response.onError(Status.CANCELLED.asRuntimeException()); + executor.shutdown(); + release.countDown(); + request.get(1, TimeUnit.SECONDS); + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + verifyNoInteractions(wrapper); + assertEquals(2L, executor.getCompletedTaskCount()); + } finally { + response.onCompleted(); + release.countDown(); + executor.shutdownNow(); + caller.join(1000); + } + } + + @Test(timeout = 5000) + public void testOneShotObservesContextCancellationAndClosesIterator() { + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + ScanIterator iterator = mock(ScanIterator.class); + StreamObserver output = mock(StreamObserver.class); + Context.CancellableContext context = Context.current().withCancellation(); + when(wrapper.scanAll(anyString(), anyString(), any(byte[].class))).thenReturn(iterator); + when(iterator.hasNext()).thenAnswer(invocation -> { + context.cancel(null); + return true; + }); + when(iterator.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + when(iterator.position()).thenReturn(new byte[4]); + try { + context.run(() -> ScanOneShotResponse.scanOneShot( + ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL).setLimit(100).build(), + output, wrapper)); + verify(iterator).close(); + verifyNoInteractions(output); + } finally { + context.cancel(null); + } + } + + @Test + public void testPausedBatchClosesIteratorAndRejectsLaterQuery() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + ScanIterator iterator = mock(ScanIterator.class); + StreamObserver output = mock(StreamObserver.class); + ScanBatchResponse response = new ScanBatchResponse(output, wrapper, executor); + Field field = ScanBatchResponse.class.getDeclaredField("iterator"); + field.setAccessible(true); + field.set(response, iterator); + try { + response.onCompleted(); + response.onNext(ScanStreamBatchReq.newBuilder() + .setQueryRequest(ScanQueryRequest.getDefaultInstance()) + .build()); + response.onCompleted(); + verify(iterator).close(); + verifyNoInteractions(wrapper); + assertNull(field.get(response)); + assertFalse(executor.isShutdown()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testRejectedBatchReportsFailureWithoutSuccessfulCompletion() { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + executor.shutdown(); + StreamObserver output = mock(StreamObserver.class); + ScanBatchResponse response = new ScanBatchResponse(output, mock(HgStoreWrapperEx.class), executor); + response.onNext(ScanStreamBatchReq.newBuilder() + .setQueryRequest(ScanQueryRequest.getDefaultInstance()).build()); + verify(output).onError(any(Throwable.class)); + verify(output, never()).onCompleted(); + } + + @Test + public void testBatchIteratorFailureClosesResourcesWithoutSuccessfulCompletion() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + StreamObserver output = mock(StreamObserver.class); + ScanIterator iterator = mock(ScanIterator.class); + IllegalStateException failure = new IllegalStateException("iterator failed"); + when(iterator.hasNext()).thenThrow(failure); + ScanBatchResponse response = new ScanBatchResponse(output, mock(HgStoreWrapperEx.class), executor); + Field field = ScanBatchResponse.class.getDeclaredField("iterator"); + field.setAccessible(true); + field.set(response, iterator); + Method send = ScanBatchResponse.class.getDeclaredMethod("sendEntries"); + send.setAccessible(true); + try { + send.invoke(response); + verify(iterator).close(); + verify(output).onError(failure); + verify(output, never()).onCompleted(); + } finally { + response.onCompleted(); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testActiveStreamCancellationReleasesBlockedProducer() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + HgStoreWrapperEx wrapper = mock(HgStoreWrapperEx.class); + ScanIterator iterator = mock(ScanIterator.class); + when(wrapper.scanAll(anyString(), anyString(), any(byte[].class))).thenReturn(iterator); + when(iterator.hasNext()).thenReturn(true); + when(iterator.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + when(iterator.position()).thenReturn(new byte[4]); + AppConfig config = mock(AppConfig.class); + when(config.getServerWaitTime()).thenReturn(60); + StreamObserver output = mock(StreamObserver.class); + ScanStreamResponse response = ScanStreamResponse.of(output, wrapper, executor, config); + try { + // Consume just one page, then leave the producer waiting for its consumer. + response.onNext(ScanStreamReq.newBuilder().setMethod(ScanMethod.ALL) + .setPageSize(1).setLimit(1000).build()); + response.onError(Status.CANCELLED.asRuntimeException()); + executor.shutdown(); + assertTrue("cancel must release a producer without its 60-second timeout", + executor.awaitTermination(1, TimeUnit.SECONDS)); + verify(iterator).close(); + } finally { + response.onCompleted(); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testParallelPausedScannerReleasesIterator() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScanIterator source = mock(ScanIterator.class); + when(source.hasNext()).thenReturn(true); + when(source.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + ParallelScanIterator scan = ParallelScanIterator.of( + () -> new KVPair<>(mock(QueryCondition.class), source), () -> Long.MAX_VALUE, + ScanQueryRequest.getDefaultInstance(), executor); + try { + // One worker fills its output allowance and pauses while retaining its iterator. + executor.submit(() -> { }).get(2, TimeUnit.SECONDS); + verify(source, atLeastOnce()).next(); + verify(source, never()).close(); + scan.close(); + scan.close(); + verify(source).close(); + executor.shutdown(); + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + } finally { + scan.close(); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testParallelOrderedScannerReleasesQueueLockOnEmptyIterator() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScanIterator first = mock(ScanIterator.class); + ScanIterator empty = mock(ScanIterator.class); + when(first.hasNext()).thenReturn(true); + when(first.next()).thenReturn(RocksDBSession.BackendColumn.of(new byte[4], new byte[0])); + when(empty.hasNext()).thenReturn(false); + + AtomicInteger supplies = new AtomicInteger(); + Field bodySize = ParallelScanIterator.class.getDeclaredField("maxBodySize"); + bodySize.setAccessible(true); + int originalBodySize = bodySize.getInt(null); + bodySize.setInt(null, 2); + ParallelScanIterator scan = null; + try { + scan = ParallelScanIterator.of( + () -> { + int supply = supplies.incrementAndGet(); + if (supply == 1) { + return new KVPair<>(mock(QueryCondition.class), first); + } + if (supply == 2) { + return new KVPair<>(mock(QueryCondition.class), empty); + } + return new KVPair<>(mock(QueryCondition.class), null); + }, + () -> 1L, + ScanQueryRequest.newBuilder().setOrderType(ScanOrderType.ORDER_WITHIN_VERTEX).build(), + executor); + + // Wait for the scanner task to finish before checking the lock owner. + executor.submit(() -> { }).get(2, TimeUnit.SECONDS); + Field queueLock = ParallelScanIterator.class.getDeclaredField("queueLock"); + queueLock.setAccessible(true); + assertFalse(((ReentrantLock) queueLock.get(scan)).isLocked()); + } finally { + if (scan != null) { + scan.close(); + } + bodySize.setInt(null, originalBodySize); + executor.shutdownNow(); + } + } + + @Test(timeout = 5000) + public void testParallelOrderedProducerCancelsWithFullOutputQueue() throws Exception { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(1); + ScanIterator source = mock(ScanIterator.class); + Field bodySize = ParallelScanIterator.class.getDeclaredField("maxBodySize"); + bodySize.setAccessible(true); + byte[] value = new byte[bodySize.getInt(null)]; + CountDownLatch full = new CountDownLatch(1); + AtomicInteger rows = new AtomicInteger(); + when(source.hasNext()).thenReturn(true); + when(source.next()).thenAnswer(invocation -> { + if (rows.incrementAndGet() == 5) { + full.countDown(); + } + return RocksDBSession.BackendColumn.of(new byte[4], value); + }); + ParallelScanIterator scan = ParallelScanIterator.of( + () -> new KVPair<>(mock(QueryCondition.class), source), () -> Long.MAX_VALUE, + ScanQueryRequest.newBuilder().setOrderType(ScanOrderType.ORDER_WITHIN_VERTEX).build(), + executor); + try { + assertTrue("four queued batches must fill the single-scanner queue", + full.await(2, TimeUnit.SECONDS)); + scan.close(); + executor.shutdown(); + assertTrue("cancellation must release the blocked ordered producer", + executor.awaitTermination(1, TimeUnit.SECONDS)); + verify(source).close(); + } finally { + scan.close(); + executor.shutdownNow(); + } + } + + private static void assertUnavailable(Runnable action) { + try { + action.run(); + fail("Scan admission must be closed"); + } catch (StatusRuntimeException e) { + assertEquals(Status.Code.UNAVAILABLE, e.getStatus().getCode()); + } + } +} diff --git a/hugegraph-store/hg-store-rocksdb/pom.xml b/hugegraph-store/hg-store-rocksdb/pom.xml index cd9cf28c6d..232aecffa1 100644 --- a/hugegraph-store/hg-store-rocksdb/pom.xml +++ b/hugegraph-store/hg-store-rocksdb/pom.xml @@ -57,7 +57,7 @@ org.rocksdb rocksdbjni - 7.7.3 + ${rocksdb.version} org.projectlombok diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java index 7fcd07f3b8..374778d9e2 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBOptions.java @@ -56,7 +56,6 @@ public class RocksDBOptions extends OptionHolder { disallowEmpty(), false ); - // public static final ConfigListOption DATA_DISKS = // new ConfigListOption<>( // "rocksdb.data_disks", diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index f4e7605a7f..bfd3240ba7 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -450,8 +450,9 @@ private void openRocksDB(String dbDataPath, long version) { new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions)); } List columnFamilyHandleList = new ArrayList<>(); - this.rocksDB = RocksDB.open(dbOptions, dbPath, columnFamilyDescriptorList, - columnFamilyHandleList); + this.rocksDB = + RocksDB.open(dbOptions, dbPath, columnFamilyDescriptorList, + columnFamilyHandleList); Asserts.isTrue(columnFamilyHandleList.size() > 0, "must have column family"); for (ColumnFamilyHandle handle : columnFamilyHandleList) { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java index 3e8545447b..9ec5750bb9 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/raft/HgStoreStateMachineTest.java @@ -130,6 +130,21 @@ public void testOnApply() { final java.util.Iterator iterator = tasks.iterator(); Task task; + @Override + public void setAutoCommitPerLog(boolean autoCommit) { + throw new UnsupportedOperationException("Test iterator does not manage commits"); + } + + @Override + public boolean commit() { + throw new UnsupportedOperationException("Test iterator does not manage commits"); + } + + @Override + public void commitAndSnapshotSync(Closure done) { + throw new UnsupportedOperationException("Test iterator does not manage snapshots"); + } + @Override public ByteBuffer getData() { return task.getData(); @@ -155,16 +170,6 @@ public void setErrorAndRollback(long ntail, Status st) { } - @Override - public boolean commit() { - throw new UnsupportedOperationException(); - } - - @Override - public void commitAndSnapshotSync(Closure done) { - throw new UnsupportedOperationException(); - } - @Override public boolean hasNext() { return iterator.hasNext(); diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java index 4bf691a6cd..22a971ebf6 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/snapshot/HgSnapshotHandlerTest.java @@ -334,23 +334,29 @@ public void testDbCompactionSkipsWhenRangeLockStillHeldAfterWait() throws Interr * until the 6-hour path-lock timeout. Interrupts a real compactionPool worker thread while * it is parked in tryLock() (identified by stack trace, since the pool is shared), then * confirms the path lock returns to its available state while the snapshot range lock - * remains owned by this test. + * remains owned by this test. Then verifies that another compaction can publish a snapshot. */ @Test public void testDbCompactionReleasesPathLockWhenInterruptedWaitingForRangeLock() throws InterruptedException { BusinessHandler businessHandler = getStoreEngine().getBusinessHandler(); int partitionId = 5; - createPartitionEngine(partitionId); + PartitionEngine engine = createPartitionEngine(partitionId); + File snapshotDirectory = new File(engine.getOptions().getRaftSnapShotPath(), "snapshot"); + File[] initialSnapshots = snapshotDirectory.listFiles( + (dir, name) -> name.startsWith("snapshot_")); + assertTrue("this partition must not have a snapshot before compaction", + initialSnapshots == null || initialSnapshots.length == 0); long originalWaitMillis = BusinessHandlerImpl.getCompactionRangeLockWaitMillis(); // Long enough that the worker thread is still parked in tryLock() when interrupted, // rather than racing a real timeout. BusinessHandlerImpl.setCompactionRangeLockWaitMillis(60_000); + boolean rangeLockHeld = false; try { // Simulate a snapshot save that is still in progress, forcing dbCompaction() onto // the tryLock(wait) path rather than acquiring the range lock immediately. - assertTrue("snapshot save must reserve the range lock", - businessHandler.tryLockCompactionRange(partitionId)); + rangeLockHeld = businessHandler.tryLockCompactionRange(partitionId); + assertTrue("snapshot save must reserve the range lock", rangeLockHeld); businessHandler.dbCompaction("graph0", partitionId); @@ -373,7 +379,7 @@ public void testDbCompactionReleasesPathLockWhenInterruptedWaitingForRangeLock() other.interrupt(); // Wait for the interrupted task to release the path lock. No second compaction - // runs here, so the available state remains stable. + // has started yet, so the available state remains stable during this wait. long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (pathLockBeforeInterrupt.get() != BusinessHandler.compactionCanStart && System.nanoTime() < deadline) { @@ -399,8 +405,41 @@ public void testDbCompactionReleasesPathLockWhenInterruptedWaitingForRangeLock() rangeLockCheck.join(); assertFalse("range lock must still belong to the snapshot save", concurrentRangeLockResult.get()); - } finally { + + // The path lock, however, must have been released by the interrupted task - + // otherwise this second dbCompaction() call would block on lock(path) until the + // 6-hour timeout instead of reaching the compacting state below once the range + // lock is released. Ownership of the range lock passes to the second + // dbCompaction() call's own worker thread, which acquires and releases it itself - + // do not touch compactionRangeLock again after this point. businessHandler.unlockCompactionRange(partitionId); + rangeLockHeld = false; + businessHandler.dbCompaction("graph0", partitionId); + + // compactionDone is transient: the following Raft snapshot resets it. + // A newly published snapshot proves the second task passed both locks + // and compacted the DB; the released path lock proves its callback finished. + // Allow for doSnapshotSync's bounded 5-second wait before saving. + long snapshotDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(15); + boolean completed = false; + while (System.nanoTime() < snapshotDeadline) { + File[] snapshots = snapshotDirectory.listFiles((dir, name) -> + name.startsWith("snapshot_") && + new File(new File(dir, name), "__raft_snapshot_meta").isFile()); + completed = snapshots != null && snapshots.length > 0 && + businessHandler.getPathLockState(path).get() == + BusinessHandler.compactionCanStart; + if (completed) { + break; + } + Thread.sleep(50); + } + assertTrue("second dbCompaction() must publish a snapshot and release the " + + "path lock after the interrupted task", completed); + } finally { + if (rangeLockHeld) { + businessHandler.unlockCompactionRange(partitionId); + } BusinessHandlerImpl.setCompactionRangeLockWaitMillis(originalWaitMillis); } } diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index b721884ff0..771ec4b76e 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -266,8 +266,7 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/com.alipay.sofa/bolt/1.6.4 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa/hessian/3.3.6 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa/hessian/3.3.7 -> Apache 2.0 - https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.11 -> Apache 2.0 - https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.13 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alipay.sofa/jraft-core/1.3.14 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa/sofa-rpc-all/5.7.6 -> Apache 2.0 https://central.sonatype.com/artifact/com.alipay.sofa/tracer-core/3.0.8 -> Apache 2.0 https://central.sonatype.com/artifact/com.beust/jcommander/1.30 -> Apache 2.0 @@ -480,6 +479,11 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/net.java.dev.jna/jna/5.12.1 -> Apache 2.0 https://central.sonatype.com/artifact/net.java.dev.jna/jna/5.5.0 -> Apache 2.0 https://central.sonatype.com/artifact/net.java.dev.jna/jna/5.7.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.github.jnr/jnr-ffi/2.1.7 -> Apache 2.0 + https://central.sonatype.com/artifact/joda-time/joda-time/2.10.8 -> Apache 2.0 + https://central.sonatype.com/artifact/org.skyscreamer/jsonassert/1.5.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.jayway.jsonpath/json-path/2.5.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.googlecode.json-simple/json-simple/1.1 -> Apache 2.0 https://central.sonatype.com/artifact/net.jodah/failsafe/2.4.1 -> Apache 2.0 https://central.sonatype.com/artifact/net.minidev/accessors-smart/1.2 -> Apache 2.0 https://central.sonatype.com/artifact/net.minidev/json-smart/2.3 -> Apache 2.0 @@ -654,10 +658,48 @@ non-Apache-2.0 license text is required for a bundled component. https://central.sonatype.com/artifact/org.powermock/powermock-module-junit4-rule/2.0.0-RC.3 -> Apache 2.0 https://central.sonatype.com/artifact/org.powermock/powermock-module-junit4/2.0.0-RC.3 -> Apache 2.0 https://central.sonatype.com/artifact/org.powermock/powermock-reflect/2.0.0-RC.3 -> Apache 2.0 + https://central.sonatype.com/artifact/com.google.api.grpc/proto-google-common-protos/1.17.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.google.api.grpc/proto-google-common-protos/2.0.1 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-api/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-collectionschema/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-core/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.protostuff/protostuff-runtime/1.6.0 -> Apache 2.0 + https://central.sonatype.com/artifact/com.addthis.metrics/reporter-config3/3.0.3 -> Apache 2.0 + https://central.sonatype.com/artifact/com.addthis.metrics/reporter-config-base/3.0.3 -> Apache 2.0 + https://central.sonatype.com/artifact/org.opencypher/rewriting-9.0/9.0.20190305 -> Apache 2.0 + https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/8.10.2 -> Apache 2.0 + https://github.com/hugegraph/toplingdb/packages/3151853 -> Apache 2.0 + https://central.sonatype.com/artifact/org.fusesource/sigar/1.6.4 -> Apache 2.0 + https://central.sonatype.com/artifact/io.prometheus/simpleclient/0.10.0 -> Apache 2.0 + https://central.sonatype.com/artifact/io.prometheus/simpleclient_common/0.10.0 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-agent/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-cli/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-cli/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-core/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-core/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-hflame/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-jfr5/0.5 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-jfr6/0.7 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-jfr-standalone/0.7 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-json/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-json/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.perfkit.sjk.parsers/sjk-nps/0.9 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-stacktrace/0.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.gridkit.jvmtool/sjk-stacktrace/0.22 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.18 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.26 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.27 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/1.28 -> Apache 2.0 + https://central.sonatype.com/artifact/org.yaml/snakeyaml/2.2 -> Apache 2.0 + https://central.sonatype.com/artifact/org.xerial.snappy/snappy-java/1.1.2.6 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alipay.sofa.common/sofa-common-tools/1.0.12 -> Apache 2.0 + https://central.sonatype.com/artifact/com.alipay.sofa/sofa-rpc-all/5.7.6 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework/spring-aop/5.3.20 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework/spring-beans/5.3.20 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework.boot/spring-boot/2.5.14 -> Apache 2.0 + https://central.sonatype.com/artifact/org.springframework.boot/spring-boot-actuator/2.5.14 -> Apache 2.0 https://central.sonatype.com/artifact/org.roaringbitmap/RoaringBitmap/0.9.38 -> Apache 2.0 https://central.sonatype.com/artifact/org.roaringbitmap/shims/0.9.38 -> Apache 2.0 - https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/6.29.5 -> Apache 2.0 - https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/7.7.3 -> Apache 2.0 https://central.sonatype.com/artifact/org.rocksdb/rocksdbjni/8.10.2 -> Apache 2.0 https://central.sonatype.com/artifact/org.skyscreamer/jsonassert/1.5.0 -> Apache 2.0 https://central.sonatype.com/artifact/org.springframework.boot/spring-boot-actuator-autoconfigure/2.5.14 -> Apache 2.0 diff --git a/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt b/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-rocksdbjni-8.10.2-SNAPSHOT.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/install-dist/scripts/build-topling-distribution.sh b/install-dist/scripts/build-topling-distribution.sh new file mode 100755 index 0000000000..8d8d2b4e06 --- /dev/null +++ b/install-dist/scripts/build-topling-distribution.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR"/../.. && pwd)" +COMPONENT="${1:-}" +VERSION="${2:-}" + +if [ -z "$COMPONENT" ]; then + echo "Usage: $0 [version]" >&2 + exit 1 +fi +if [ "$(uname -s)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then + echo "Error: Topling distributions can only be prepared on Linux x86_64" >&2 + exit 1 +fi +if [ -z "$VERSION" ]; then + VERSION=$(mvn -f "$PROJECT_ROOT/pom.xml" \ + help:evaluate -Dexpression=project.version -q -DforceStdout) +fi +if [[ ! "$VERSION" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then + echo "Error: unsafe project version: $VERSION" >&2 + exit 1 +fi + +case "$COMPONENT" in + server) + COMPONENT_PARENT="$PROJECT_ROOT/hugegraph-server" + STANDARD_NAME="apache-hugegraph-server-$VERSION" + CONFIG_FILE="conf/graphs/hugegraph.properties" + ;; + pd) + COMPONENT_PARENT="$PROJECT_ROOT/hugegraph-pd" + STANDARD_NAME="apache-hugegraph-pd-$VERSION" + CONFIG_FILE="conf/application.yml" + ;; + store) + COMPONENT_PARENT="$PROJECT_ROOT/hugegraph-store" + STANDARD_NAME="apache-hugegraph-store-$VERSION" + CONFIG_FILE="conf/application-pd.yml" + ;; + *) + echo "Error: unsupported component '$COMPONENT'" >&2 + exit 1 + ;; +esac + +STANDARD_DIR="$COMPONENT_PARENT/$STANDARD_NAME" +TOPLING_NAME="$STANDARD_NAME-topling" +TOPLING_DIR="$COMPONENT_PARENT/$TOPLING_NAME" +TOPLING_TAR="$COMPONENT_PARENT/$TOPLING_NAME.tar.gz" +TOPLING_SOURCE_LIB="$PROJECT_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/lib/topling" +TOPLING_JARS=("$TOPLING_SOURCE_LIB"/rocksdbjni*.jar) + +if [ ! -d "$STANDARD_DIR" ]; then + echo "Error: standard distribution not found: $STANDARD_DIR" >&2 + exit 1 +fi +if ! command -v rsync >/dev/null 2>&1; then + echo "Error: rsync is required to build a clean Topling distribution" >&2 + exit 1 +fi +if [ "${#TOPLING_JARS[@]}" -ne 1 ] || [ ! -f "${TOPLING_JARS[0]}" ]; then + echo "Error: expected exactly one ToplingDB JAR under: $TOPLING_SOURCE_LIB" >&2 + exit 1 +fi + +STAGING_ROOT="$(mktemp -d "$COMPONENT_PARENT/.topling-dist.XXXXXX")" +cleanup() { + rm -rf "$STAGING_ROOT" +} +trap cleanup EXIT +STAGING_DIR="$STAGING_ROOT/$TOPLING_NAME" + +mkdir -p "$STAGING_DIR" +rsync --archive \ + --exclude='/bin/pid' \ + --exclude='/library' \ + --exclude='/logs' \ + --exclude='/pd_data' \ + --exclude='/rocksdb-data' \ + --exclude='/storage' \ + "$STANDARD_DIR/" "$STAGING_DIR/" + +for runtime_path in \ + bin/pid \ + library \ + logs \ + pd_data \ + rocksdb-data \ + storage; do + if [ -e "$STAGING_DIR/$runtime_path" ] || + [ -L "$STAGING_DIR/$runtime_path" ]; then + echo "Error: runtime state leaked into staging: $runtime_path" >&2 + exit 1 + fi +done + +mkdir -p "$STAGING_DIR/lib/topling" +cp "${TOPLING_JARS[0]}" "$STAGING_DIR/lib/topling/" + +case "$COMPONENT" in + server) + sed -i '/^[[:space:]#]*rocksdb\.provider[[:space:]]*=/d' \ + "$STAGING_DIR/$CONFIG_FILE" + printf '\nrocksdb.provider=topling\n' >> "$STAGING_DIR/$CONFIG_FILE" + ;; + pd) + sed -i \ + -e 's/^# rocksdb:/rocksdb:/' \ + -e 's/^# provider: rocksdb/ provider: topling/' \ + "$STAGING_DIR/$CONFIG_FILE" + ;; + store) + sed -i 's/^ # provider: topling/ provider: topling/' \ + "$STAGING_DIR/$CONFIG_FILE" + ;; +esac + +if ! grep -Eq '^[[:space:]]*(rocksdb\.provider=|provider:[[:space:]]*)topling([[:space:]]|$)' \ + "$STAGING_DIR/$CONFIG_FILE"; then + echo "Error: failed to select ToplingDB in $STAGING_DIR/$CONFIG_FILE" >&2 + exit 1 +fi + +"$STAGING_DIR/bin/prepare-topling.sh" + +NATIVE_LIBRARY="$STAGING_DIR/library/librocksdbjni-linux64.so" +if [ ! -r "$NATIVE_LIBRARY" ]; then + echo "Error: prepared native library not found: $NATIVE_LIBRARY" >&2 + exit 1 +fi +if command -v ldd >/dev/null 2>&1 && + ldd "$NATIVE_LIBRARY" 2>/dev/null | grep -q 'not found'; then + echo "Error: prepared ToplingDB native library has unresolved dependencies" >&2 + ldd "$NATIVE_LIBRARY" >&2 || true + exit 1 +fi + +JAR_SHA256=$(sha256sum "${TOPLING_JARS[0]}" | awk '{ print $1 }') +printf '%s\n' \ + "provider=topling" \ + "component=$COMPONENT" \ + "version=$VERSION" \ + "jar=$(basename "${TOPLING_JARS[0]}")" \ + "jar.sha256=$JAR_SHA256" \ + > "$STAGING_DIR/lib/topling/runtime.properties" + +tar -czf "$STAGING_ROOT/$TOPLING_NAME.tar.gz" \ + -C "$STAGING_ROOT" "$TOPLING_NAME" + +if [ -e "$TOPLING_DIR" ] && + [[ "$TOPLING_DIR" != "$COMPONENT_PARENT"/apache-hugegraph-*-topling ]]; then + echo "Error: refusing to replace unexpected path: $TOPLING_DIR" >&2 + exit 1 +fi +rm -rf "$TOPLING_DIR" +rm -f "$TOPLING_TAR" +mv "$STAGING_DIR" "$TOPLING_DIR" +mv "$STAGING_ROOT/$TOPLING_NAME.tar.gz" "$TOPLING_TAR" + +echo "[build-topling-distribution] Directory: $TOPLING_DIR" +echo "[build-topling-distribution] Archive: $TOPLING_TAR" diff --git a/install-dist/scripts/dependency/known-dependencies.txt b/install-dist/scripts/dependency/known-dependencies.txt index 29151532a6..bfa90d794e 100644 --- a/install-dist/scripts/dependency/known-dependencies.txt +++ b/install-dist/scripts/dependency/known-dependencies.txt @@ -282,8 +282,7 @@ jna-5.5.0.jar jna-5.7.0.jar joda-time-2.10.8.jar joni-2.2.1.jar -jraft-core-1.3.11.jar -jraft-core-1.3.13.jar +jraft-core-1.3.14.jar jsonassert-1.5.0.jar json-path-2.5.0.jar json-smart-2.3.jar @@ -464,8 +463,6 @@ protostuff-core-1.6.0.jar protostuff-runtime-1.6.0.jar rewriting-9.0-9.0.20190305.jar RoaringBitmap-0.9.38.jar -rocksdbjni-6.29.5.jar -rocksdbjni-7.7.3.jar rocksdbjni-8.10.2.jar scala-java8-compat_2.12-0.8.0.jar scala-library-2.12.7.jar @@ -487,7 +484,6 @@ slf4j-api-1.7.21.jar slf4j-api-1.7.25.jar slf4j-api-1.7.31.jar slf4j-api-1.7.32.jar -slf4j-api-2.0.9.jar snakeyaml-1.18.jar snakeyaml-1.26.jar snakeyaml-1.27.jar diff --git a/pom.xml b/pom.xml index 65dee5a154..a01296ce98 100644 --- a/pom.xml +++ b/pom.xml @@ -89,15 +89,16 @@ 5.6.0 1.7.0 1.18.30 - - 1.3.11 - 1.3.13 + + 1.3.14 + 1.3.14 hugegraph 11 11 UTF-8 bash 1.5.0 + 8.10.2 @@ -203,6 +204,8 @@ .gitattributes .serena/** + + .goal-task/** **/*.iml **/*.iws @@ -221,6 +224,9 @@ **/src/main/java/org/apache/hugegraph/pd/grpc/** **/src/main/java/org/apache/hugegraph/store/grpc/** + + **/library/*.html + **/library/*.css true