From 2751eb94738b6608871c2acdf4b19cd3e3c4d091 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 2 Oct 2026 17:42:14 +0530 Subject: [PATCH 01/23] fix(server): keep Server crash diagnostics after a container restart When a hugegraph/server container exited during startup, kubectl logs showed only "Starting HugeGraphServer failed". The cause stayed in logs/ inside the container, and Kubernetes dropped it on restart. - Set STDOUT_MODE=true in Dockerfile-hstore. #2980 did this for the standalone Dockerfile and missed this one. - In conf/log4j2.xml, also send WARN and above from the hadoop, zookeeper, sofa, netty and commons loggers to console. They were file-only. - In hugegraph-server.sh, always set -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath and -XX:ErrorFile under $LOGS. The heap-dump flags used to be dropped whenever JAVA_OPTIONS was set, and hs_err files went to the install directory. The file names carry the launch time because HotSpot will not overwrite an existing crash log or heap dump, and a restarted container often reuses the PID. The flags go before JAVA_OPTIONS and -j, so values given there still win; a flag already set in JAVA_TOOL_OPTIONS or JDK_JAVA_OPTIONS, which the JVM reads before the command line, is left out. - Say "container logs" rather than "docker logs" in the start failure message, since the same image runs on Kubernetes. - Cover the new flags and their overrides in test-java-security-properties.sh, and document the log directory and the Kubernetes mount in docker/README.md. This applies to every Server install, bare metal included: an OOM now writes a heap dump to logs/ even when JAVA_OPTIONS is set. Fixes #3256 --- docker/README.md | 6 +++ hugegraph-server/Dockerfile-hstore | 3 +- .../assembly/static/bin/hugegraph-server.sh | 32 ++++++++++- .../assembly/static/bin/start-hugegraph.sh | 2 +- .../src/assembly/static/conf/log4j2.xml | 5 ++ .../travis/test-java-security-properties.sh | 53 +++++++++++++++++++ 6 files changed, 98 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index a9d43dcb1f..e983ce31ee 100644 --- a/docker/README.md +++ b/docker/README.md @@ -230,6 +230,12 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t `docker compose down` keeps named-volume data. `docker compose down -v` intentionally deletes it. +### Server logs and crash files + +Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout; their INFO output, the audit log and the slow-query log stay in files only. When a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. + +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid_.log`) and out-of-memory heap dumps (`java_pid_.hprof`). `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap; size the volume for it, or pass `-XX:-HeapDumpOnOutOfMemoryError` in `JAVA_OPTS` to turn heap dumps off. + ## Developers ### Images and Compose files diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index e90979563b..2e2919248a 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -41,7 +41,8 @@ LABEL maintainer="HugeGraph Docker Maintainers " # TODO: use g1gc or zgc as default ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm" \ - HUGEGRAPH_HOME="hugegraph-server" + HUGEGRAPH_HOME="hugegraph-server" \ + STDOUT_MODE="true" WORKDIR /hugegraph-server/ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index caffedc482..376abee67a 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -121,13 +121,43 @@ if [ "$JAVA_OPTIONS" = "" ]; then echo "Failed to start HugeGraphServer, requires at least ${MIN_MEM}MB free memory" >> "${OUTPUT}" exit 1 fi - JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}" + JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m ${USER_OPTION}" # Rolling out detailed GC logs #JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseGCLogFileRotation -XX:GCLogFileSize=10M -XX:NumberOfGCLogFiles=3 \ # -Xloggc:./logs/gc.log -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps" fi +# Keep heap dumps and JVM crash logs in $LOGS whatever JAVA_OPTIONS holds. They go +# first so a value from JAVA_OPTIONS, -j or _JAVA_OPTIONS, all later on the command +# line, still wins. JAVA_TOOL_OPTIONS and JDK_JAVA_OPTIONS are read before the command +# line, so a flag set there is left out here instead of being overridden. +# A restarted container often reuses the JVM's PID, and HotSpot will not overwrite an +# existing crash log or heap dump, so the file names also carry the launch time. The +# script execs java below, so $$ is the JVM's PID. +LAUNCH_STAMP=$(date +%Y%m%d-%H%M%S) +CRASH_OPTIONS="" +SET_HEAP_DUMP="true" +SET_HEAP_DUMP_PATH="true" +SET_ERROR_FILE="true" +for OPTION in ${JAVA_TOOL_OPTIONS:-} ${JDK_JAVA_OPTIONS:-}; do + case "${OPTION}" in + -XX:[+-]HeapDumpOnOutOfMemoryError) SET_HEAP_DUMP="false" ;; + -XX:HeapDumpPath=*) SET_HEAP_DUMP_PATH="false" ;; + -XX:ErrorFile=*) SET_ERROR_FILE="false" ;; + esac +done +if [[ $SET_HEAP_DUMP == "true" ]]; then + CRASH_OPTIONS="${CRASH_OPTIONS} -XX:+HeapDumpOnOutOfMemoryError" +fi +if [[ $SET_HEAP_DUMP_PATH == "true" ]]; then + CRASH_OPTIONS="${CRASH_OPTIONS} -XX:HeapDumpPath=${LOGS}/java_pid$$_${LAUNCH_STAMP}.hprof" +fi +if [[ $SET_ERROR_FILE == "true" ]]; then + CRASH_OPTIONS="${CRASH_OPTIONS} -XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_STAMP}.log" +fi +JAVA_OPTIONS="${CRASH_OPTIONS} ${JAVA_OPTIONS}" + if [[ $JAVA_VERSION -gt 9 ]]; then JAVA_OPTIONS="${JAVA_OPTIONS} --add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED \ --add-modules=jdk.unsupported \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 2c99238327..f7997cc13c 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -112,7 +112,7 @@ if [[ $DAEMON == "true" ]]; then wait_for_startup ${PID} 'HugeGraphServer' "$REST_SERVER_URL/graphs" "${SERVER_STARTUP_TIMEOUT_S}" || { if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - echo "See 'docker logs' for HugeGraphServer log output." >&2 + echo "See the container logs ('docker logs' or 'kubectl logs') for HugeGraphServer log output." >&2 else echo "See $LOGS/hugegraph-server.log for HugeGraphServer log output." >&2 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/log4j2.xml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/log4j2.xml index 6286314d7b..bc40d2955e 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/log4j2.xml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/log4j2.xml @@ -108,18 +108,23 @@ + + + + + diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 796d69c83b..6ec19f5cc6 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -35,6 +35,13 @@ assert_argument() { fail "missing JVM argument: $argument" } +assert_argument_matching() { + local pattern="$1" + local capture="$2" + grep -Eq -- "$pattern" "$capture" || \ + fail "missing JVM argument matching: $pattern" +} + assert_no_argument() { local pattern="$1" local capture="$2" @@ -432,6 +439,52 @@ assert_argument "true" "$ENABLED_CAPTURE" assert_argument "-Doperator.marker=preserved" "$ENABLED_CAPTURE" assert_no_argument '^-D(networkaddress\.cache\.ttl|sun\.net\.inetaddr\.ttl)=' \ "$ENABLED_CAPTURE" +assert_argument "-XX:+HeapDumpOnOutOfMemoryError" "$ENABLED_CAPTURE" +# The names carry the launch time, since a restarted container often reuses the +# PID and HotSpot will not overwrite an existing crash log or heap dump. +LOGS_PATTERN=$(printf '%s' "${SERVER_ROOT}/logs" | sed 's/[][\.*^$+?(){}|]/\\&/g') +assert_argument_matching \ + "^-XX:HeapDumpPath=${LOGS_PATTERN}/java_pid[0-9]+_[0-9]{8}-[0-9]{6}\.hprof$" \ + "$ENABLED_CAPTURE" +assert_argument_matching \ + "^-XX:ErrorFile=${LOGS_PATTERN}/hs_err_pid%p_[0-9]{8}-[0-9]{6}\.log$" \ + "$ENABLED_CAPTURE" + +# Heap dumps and JVM crash logs must stay in logs/ when JAVA_OPTIONS replaces +# the default heap options, and an operator's own paths must still win. +CUSTOM_OPTIONS_CAPTURE="${TEMP_DIR}/custom-java-options.args" +CAPTURE_FILE="$CUSTOM_OPTIONS_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + JAVA_OPTIONS="-Xmx1g -XX:ErrorFile=/operator/hs_err.log \ + -XX:HeapDumpPath=/operator/dumps" \ + STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null + +assert_argument "-Xmx1g" "$CUSTOM_OPTIONS_CAPTURE" +assert_argument "-XX:+HeapDumpOnOutOfMemoryError" "$CUSTOM_OPTIONS_CAPTURE" +assert_argument_matching "^-XX:HeapDumpPath=${LOGS_PATTERN}/" "$CUSTOM_OPTIONS_CAPTURE" +LAST_ERROR_FILE_ARGUMENT=$(grep -E '^-XX:ErrorFile=' \ + "$CUSTOM_OPTIONS_CAPTURE" | tail -n 1) +if [[ "$LAST_ERROR_FILE_ARGUMENT" != "-XX:ErrorFile=/operator/hs_err.log" ]]; then + fail "operator -XX:ErrorFile was overridden by the launcher default" +fi +LAST_HEAP_DUMP_PATH_ARGUMENT=$(grep -E '^-XX:HeapDumpPath=' \ + "$CUSTOM_OPTIONS_CAPTURE" | tail -n 1) +if [[ "$LAST_HEAP_DUMP_PATH_ARGUMENT" != "-XX:HeapDumpPath=/operator/dumps" ]]; then + fail "operator -XX:HeapDumpPath was overridden by the launcher default" +fi + +# JAVA_TOOL_OPTIONS and JDK_JAVA_OPTIONS are read before the command line, so +# the launcher must leave out a flag already set there rather than override it. +TOOL_OPTIONS_CAPTURE="${TEMP_DIR}/tool-java-options.args" +CAPTURE_FILE="$TOOL_OPTIONS_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + JAVA_TOOL_OPTIONS="-XX:ErrorFile=/operator/hs_err.log" \ + JDK_JAVA_OPTIONS="-XX:-HeapDumpOnOutOfMemoryError" \ + STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null + +assert_no_argument '^-XX:ErrorFile=' "$TOOL_OPTIONS_CAPTURE" +assert_no_argument '^-XX:[+-]HeapDumpOnOutOfMemoryError$' "$TOOL_OPTIONS_CAPTURE" +assert_argument_matching "^-XX:HeapDumpPath=${LOGS_PATTERN}/" "$TOOL_OPTIONS_CAPTURE" JDK21_CAPTURE="${TEMP_DIR}/jdk21.args" CAPTURE_FILE="$JDK21_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ From ed28f47158891f0fc6ce1238c23824df5b68b508 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sat, 3 Oct 2026 14:52:14 +0530 Subject: [PATCH 02/23] fix(server): keep JAVA_TOOL_OPTIONS when adding the telemetry agent With telemetry on, the launcher replaced JAVA_TOOL_OPTIONS with the OpenTelemetry -javaagent option. The crash-file defaults leave out any flag already set in JAVA_TOOL_OPTIONS, so an operator's -XX:ErrorFile there was dropped twice and HotSpot wrote hs_err to the working directory. Append the agent to the existing value instead, and cover the telemetry path in test-java-security-properties.sh with a stubbed agent checksum. --- .../assembly/static/bin/hugegraph-server.sh | 4 ++- .../travis/test-java-security-properties.sh | 36 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 376abee67a..9e06ccdd5c 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -266,7 +266,9 @@ if [ "${OPEN_TELEMETRY}" == "true" ]; then fi # Note: check carefully if multi "javeagent" params are set - export JAVA_TOOL_OPTIONS="-javaagent:${PLUGINS}/${OT_JAR}" + # Keep the operator's JAVA_TOOL_OPTIONS: the crash-file defaults above leave + # out any flag already set there, so dropping it would lose that flag. + export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:+${JAVA_TOOL_OPTIONS} }-javaagent:${PLUGINS}/${OT_JAR}" export OTEL_TRACES_EXPORTER=otlp export OTEL_METRICS_EXPORTER=none export OTEL_LOGS_EXPORTER=none diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 6ec19f5cc6..4c407d8d37 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -420,6 +420,9 @@ if [[ " $* " == *" -version "* ]]; then exit 0 fi printf '%s\n' "$@" > "$CAPTURE_FILE" +if [[ -n "${TOOL_OPTIONS_CAPTURE_FILE:-}" ]]; then + printf '%s\n' "${JAVA_TOOL_OPTIONS:-}" > "$TOOL_OPTIONS_CAPTURE_FILE" +fi MOCK chmod +x "${MOCK_JAVA_HOME}/bin/java" @@ -486,6 +489,39 @@ assert_no_argument '^-XX:ErrorFile=' "$TOOL_OPTIONS_CAPTURE" assert_no_argument '^-XX:[+-]HeapDumpOnOutOfMemoryError$' "$TOOL_OPTIONS_CAPTURE" assert_argument_matching "^-XX:HeapDumpPath=${LOGS_PATTERN}/" "$TOOL_OPTIONS_CAPTURE" +# With telemetry on, the launcher adds its agent to JAVA_TOOL_OPTIONS. It must +# keep the operator's flags there, or the -XX:ErrorFile left out above is lost. +OT_EXPECTED_MD5=$(grep -E '^ *expected_md5=' "$SERVER_SCRIPT" | cut -d'"' -f2) +MOCK_MD5_BIN="${TEMP_DIR}/mock-md5-bin" +mkdir -p "$MOCK_MD5_BIN" +printf '#!/bin/bash\necho "%s $1"\n' "$OT_EXPECTED_MD5" > "${MOCK_MD5_BIN}/md5sum" +chmod +x "${MOCK_MD5_BIN}/md5sum" +OT_JAR_PATH="${SERVER_ROOT}/plugins/opentelemetry-javaagent.jar" +OT_JAR_EXISTED="false" +if [[ -e "$OT_JAR_PATH" ]]; then + OT_JAR_EXISTED="true" +else + mkdir -p "${SERVER_ROOT}/plugins" + : > "$OT_JAR_PATH" +fi +TELEMETRY_CAPTURE="${TEMP_DIR}/telemetry.args" +TELEMETRY_TOOL_OPTIONS="${TEMP_DIR}/telemetry-tool-options.txt" +CAPTURE_FILE="$TELEMETRY_CAPTURE" TOOL_OPTIONS_CAPTURE_FILE="$TELEMETRY_TOOL_OPTIONS" \ + JAVA_HOME="$MOCK_JAVA_HOME" PATH="${MOCK_MD5_BIN}:${PATH}" \ + JAVA_TOOL_OPTIONS="-XX:ErrorFile=/operator/hs_err.log" \ + STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true "" "" true \ + >/dev/null +if [[ "$OT_JAR_EXISTED" == "false" ]]; then + rm -f "$OT_JAR_PATH" +fi + +assert_no_argument '^-XX:ErrorFile=' "$TELEMETRY_CAPTURE" +grep -Eq -- '(^| )-XX:ErrorFile=/operator/hs_err\.log( |$)' "$TELEMETRY_TOOL_OPTIONS" || + fail "telemetry agent setup dropped the operator's JAVA_TOOL_OPTIONS" +grep -Eq -- '(^| )-javaagent:[^ ]*/opentelemetry-javaagent\.jar$' "$TELEMETRY_TOOL_OPTIONS" || + fail "telemetry agent was not added to JAVA_TOOL_OPTIONS" + JDK21_CAPTURE="${TEMP_DIR}/jdk21.args" CAPTURE_FILE="$JDK21_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ MOCK_JAVA_VERSION=21 STDOUT_MODE=true "$SERVER_SCRIPT" \ From ac9d98b03e483a27c0c825b50d8d80fd6687c61c Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sat, 3 Oct 2026 21:07:52 +0530 Subject: [PATCH 03/23] fix(server): let the JVM resolve crash-file option overrides Review follow-ups on the crash-file defaults: - Put the defaults at the front of JAVA_TOOL_OPTIONS instead of on the command line, and drop the shell scan of JAVA_TOOL_OPTIONS and JDK_JAVA_OPTIONS. The JVM reads the operator's JAVA_TOOL_OPTIONS, JDK_JAVA_OPTIONS, the command line and _JAVA_OPTIONS after the defaults, so any of them overrides a default. Word splitting in the shell missed quoted options and @argfiles, and could treat flag-like text inside a property value as a flag. - Add a counter to the launch time in the file names when a heap dump or crash log with that name already exists in $LOGS. A restart in the same second with the same PID no longer targets an existing file. - Test the values a real JVM resolves, covering a quoted opt-out, an @argfile in JDK_JAVA_OPTIONS, flag text in a property value, operator paths in JAVA_OPTIONS, name collisions and telemetry. The EXIT cleanup now removes the telemetry jar and log fixtures, so a failing step does not leave them in the distribution. --- .../assembly/static/bin/hugegraph-server.sh | 55 +++--- .../travis/test-java-security-properties.sh | 169 ++++++++++++------ 2 files changed, 144 insertions(+), 80 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 9e06ccdd5c..4e9ad282ef 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -128,35 +128,33 @@ if [ "$JAVA_OPTIONS" = "" ]; then # -Xloggc:./logs/gc.log -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps" fi -# Keep heap dumps and JVM crash logs in $LOGS whatever JAVA_OPTIONS holds. They go -# first so a value from JAVA_OPTIONS, -j or _JAVA_OPTIONS, all later on the command -# line, still wins. JAVA_TOOL_OPTIONS and JDK_JAVA_OPTIONS are read before the command -# line, so a flag set there is left out here instead of being overridden. -# A restarted container often reuses the JVM's PID, and HotSpot will not overwrite an -# existing crash log or heap dump, so the file names also carry the launch time. The -# script execs java below, so $$ is the JVM's PID. +# Keep heap dumps and JVM crash logs in $LOGS whatever JAVA_OPTIONS holds. The +# defaults go first in JAVA_TOOL_OPTIONS, which the JVM reads before the operator's +# own JAVA_TOOL_OPTIONS, JDK_JAVA_OPTIONS, the command line (JAVA_OPTIONS and -j) +# and _JAVA_OPTIONS, so any of those overrides them. The JVM does that parsing +# itself, including quoted options and @argfiles. +# A restarted container often reuses the JVM's PID, and HotSpot will not overwrite +# an existing crash log or heap dump, so the file names carry the launch time plus +# a counter that skips names already used in $LOGS. The script execs java below, +# so $$ is the JVM's PID. +crash_files_exist() { + local file + for file in "${LOGS}"/java_pid*_"$1".hprof "${LOGS}"/hs_err_pid*_"$1".log; do + [[ -e ${file} ]] && return 0 + done + return 1 +} LAUNCH_STAMP=$(date +%Y%m%d-%H%M%S) -CRASH_OPTIONS="" -SET_HEAP_DUMP="true" -SET_HEAP_DUMP_PATH="true" -SET_ERROR_FILE="true" -for OPTION in ${JAVA_TOOL_OPTIONS:-} ${JDK_JAVA_OPTIONS:-}; do - case "${OPTION}" in - -XX:[+-]HeapDumpOnOutOfMemoryError) SET_HEAP_DUMP="false" ;; - -XX:HeapDumpPath=*) SET_HEAP_DUMP_PATH="false" ;; - -XX:ErrorFile=*) SET_ERROR_FILE="false" ;; - esac +LAUNCH_ID="${LAUNCH_STAMP}" +LAUNCH_SUFFIX=0 +while crash_files_exist "${LAUNCH_ID}"; do + LAUNCH_SUFFIX=$((LAUNCH_SUFFIX + 1)) + LAUNCH_ID="${LAUNCH_STAMP}-${LAUNCH_SUFFIX}" done -if [[ $SET_HEAP_DUMP == "true" ]]; then - CRASH_OPTIONS="${CRASH_OPTIONS} -XX:+HeapDumpOnOutOfMemoryError" -fi -if [[ $SET_HEAP_DUMP_PATH == "true" ]]; then - CRASH_OPTIONS="${CRASH_OPTIONS} -XX:HeapDumpPath=${LOGS}/java_pid$$_${LAUNCH_STAMP}.hprof" -fi -if [[ $SET_ERROR_FILE == "true" ]]; then - CRASH_OPTIONS="${CRASH_OPTIONS} -XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_STAMP}.log" -fi -JAVA_OPTIONS="${CRASH_OPTIONS} ${JAVA_OPTIONS}" +CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" +CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${LOGS}/java_pid$$_${LAUNCH_ID}.hprof\"" +CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_ID}.log\"" +export JAVA_TOOL_OPTIONS="${CRASH_OPTIONS}${JAVA_TOOL_OPTIONS:+ ${JAVA_TOOL_OPTIONS}}" if [[ $JAVA_VERSION -gt 9 ]]; then JAVA_OPTIONS="${JAVA_OPTIONS} --add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED \ @@ -266,8 +264,7 @@ if [ "${OPEN_TELEMETRY}" == "true" ]; then fi # Note: check carefully if multi "javeagent" params are set - # Keep the operator's JAVA_TOOL_OPTIONS: the crash-file defaults above leave - # out any flag already set there, so dropping it would lose that flag. + # Append, so the crash-file defaults and the operator's JAVA_TOOL_OPTIONS stay. export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:+${JAVA_TOOL_OPTIONS} }-javaagent:${PLUGINS}/${OT_JAR}" export OTEL_TRACES_EXPORTER=otlp export OTEL_METRICS_EXPORTER=none diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 4c407d8d37..772d7864cf 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -80,6 +80,8 @@ if [[ "$JAVA_MAJOR" -ge 18 ]]; then fi TEMP_DIR=$(mktemp -d) +CRASH_NAME_FIXTURES=() +OT_FIXTURE_JAR="" SECURITY_PROPERTIES_BACKUP="${TEMP_DIR}/java-security.properties" cleanup() { @@ -90,6 +92,14 @@ cleanup() { ! -e "$SECURITY_PROPERTIES" ]]; then mv "$SECURITY_PROPERTIES_BACKUP" "$SECURITY_PROPERTIES" fi + # Fixtures the test adds to the distribution; a failed step must not leave + # them behind, or a later real telemetry start would skip its download. + if [[ -n "${OT_FIXTURE_JAR:-}" ]]; then + rm -f "$OT_FIXTURE_JAR" + fi + if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then + rm -f "${CRASH_NAME_FIXTURES[@]}" + fi rm -rf "$TEMP_DIR" } @@ -420,9 +430,7 @@ if [[ " $* " == *" -version "* ]]; then exit 0 fi printf '%s\n' "$@" > "$CAPTURE_FILE" -if [[ -n "${TOOL_OPTIONS_CAPTURE_FILE:-}" ]]; then - printf '%s\n' "${JAVA_TOOL_OPTIONS:-}" > "$TOOL_OPTIONS_CAPTURE_FILE" -fi +printf '%s\n' "${JAVA_TOOL_OPTIONS:-}" > "${CAPTURE_FILE}.tool-options" MOCK chmod +x "${MOCK_JAVA_HOME}/bin/java" @@ -442,19 +450,54 @@ assert_argument "true" "$ENABLED_CAPTURE" assert_argument "-Doperator.marker=preserved" "$ENABLED_CAPTURE" assert_no_argument '^-D(networkaddress\.cache\.ttl|sun\.net\.inetaddr\.ttl)=' \ "$ENABLED_CAPTURE" -assert_argument "-XX:+HeapDumpOnOutOfMemoryError" "$ENABLED_CAPTURE" -# The names carry the launch time, since a restarted container often reuses the -# PID and HotSpot will not overwrite an existing crash log or heap dump. +# Heap dump and crash log defaults go first in JAVA_TOOL_OPTIONS, so the JVM's +# own precedence lets every operator source override them. Check the values a +# real JVM settles on when started the way a captured launcher run would start +# it: the captured JAVA_TOOL_OPTIONS, the crash-related -XX: arguments, and +# JDK_JAVA_OPTIONS. +effective_flag() { + local capture="$1" + local flag="$2" + local jdk_java_options="${3:-}" + local args=() + local line + while IFS= read -r line; do + if [[ "$line" =~ ^-XX:([+-]HeapDumpOnOutOfMemoryError|HeapDumpPath=|ErrorFile=) ]]; then + args+=("$line") + fi + done < "$capture" + JAVA_TOOL_OPTIONS="$(cat "${capture}.tool-options")" \ + JDK_JAVA_OPTIONS="$jdk_java_options" \ + "$JAVA_BIN" ${args[@]+"${args[@]}"} -XX:+PrintFlagsFinal -version \ + 2>/dev/null | awk -v flag="$flag" '$2 == flag { print $4; exit }' +} + +assert_effective_flag() { + local capture="$1" + local flag="$2" + local pattern="$3" + local jdk_java_options="${4:-}" + local value + value=$(effective_flag "$capture" "$flag" "$jdk_java_options") + [[ "$value" =~ $pattern ]] || + fail "effective ${flag} is '${value}', expected to match ${pattern}" +} + LOGS_PATTERN=$(printf '%s' "${SERVER_ROOT}/logs" | sed 's/[][\.*^$+?(){}|]/\\&/g') -assert_argument_matching \ - "^-XX:HeapDumpPath=${LOGS_PATTERN}/java_pid[0-9]+_[0-9]{8}-[0-9]{6}\.hprof$" \ - "$ENABLED_CAPTURE" -assert_argument_matching \ - "^-XX:ErrorFile=${LOGS_PATTERN}/hs_err_pid%p_[0-9]{8}-[0-9]{6}\.log$" \ - "$ENABLED_CAPTURE" - -# Heap dumps and JVM crash logs must stay in logs/ when JAVA_OPTIONS replaces -# the default heap options, and an operator's own paths must still win. +# The names carry the launch time and, if needed, a counter, since a restarted +# container often reuses the PID and HotSpot will not overwrite an existing +# crash log or heap dump. +HEAP_DUMP_PATTERN="^${LOGS_PATTERN}/java_pid[0-9]+_[0-9]{8}-[0-9]{6}(-[0-9]+)?\.hprof$" +ERROR_FILE_PATTERN="^${LOGS_PATTERN}/hs_err_pid%p_[0-9]{8}-[0-9]{6}(-[0-9]+)?\.log$" + +assert_no_argument '^-XX:([+-]HeapDumpOnOutOfMemoryError|HeapDumpPath=|ErrorFile=)' \ + "$ENABLED_CAPTURE" +assert_effective_flag "$ENABLED_CAPTURE" HeapDumpOnOutOfMemoryError '^true$' +assert_effective_flag "$ENABLED_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" +assert_effective_flag "$ENABLED_CAPTURE" ErrorFile "$ERROR_FILE_PATTERN" + +# JAVA_OPTIONS replaces the default heap options; the crash defaults stay, and +# paths given there win. CUSTOM_OPTIONS_CAPTURE="${TEMP_DIR}/custom-java-options.args" CAPTURE_FILE="$CUSTOM_OPTIONS_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ JAVA_OPTIONS="-Xmx1g -XX:ErrorFile=/operator/hs_err.log \ @@ -463,63 +506,87 @@ CAPTURE_FILE="$CUSTOM_OPTIONS_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null assert_argument "-Xmx1g" "$CUSTOM_OPTIONS_CAPTURE" -assert_argument "-XX:+HeapDumpOnOutOfMemoryError" "$CUSTOM_OPTIONS_CAPTURE" -assert_argument_matching "^-XX:HeapDumpPath=${LOGS_PATTERN}/" "$CUSTOM_OPTIONS_CAPTURE" -LAST_ERROR_FILE_ARGUMENT=$(grep -E '^-XX:ErrorFile=' \ - "$CUSTOM_OPTIONS_CAPTURE" | tail -n 1) -if [[ "$LAST_ERROR_FILE_ARGUMENT" != "-XX:ErrorFile=/operator/hs_err.log" ]]; then - fail "operator -XX:ErrorFile was overridden by the launcher default" -fi -LAST_HEAP_DUMP_PATH_ARGUMENT=$(grep -E '^-XX:HeapDumpPath=' \ - "$CUSTOM_OPTIONS_CAPTURE" | tail -n 1) -if [[ "$LAST_HEAP_DUMP_PATH_ARGUMENT" != "-XX:HeapDumpPath=/operator/dumps" ]]; then - fail "operator -XX:HeapDumpPath was overridden by the launcher default" -fi +assert_effective_flag "$CUSTOM_OPTIONS_CAPTURE" HeapDumpOnOutOfMemoryError '^true$' +assert_effective_flag "$CUSTOM_OPTIONS_CAPTURE" HeapDumpPath '^/operator/dumps$' +assert_effective_flag "$CUSTOM_OPTIONS_CAPTURE" ErrorFile '^/operator/hs_err\.log$' + +# The JVM, not the launcher, parses the operator's JAVA_TOOL_OPTIONS: a quoted +# opt-out works, and flag-like text inside a property value changes nothing. +QUOTED_TOOL_CAPTURE="${TEMP_DIR}/quoted-tool-options.args" +CAPTURE_FILE="$QUOTED_TOOL_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + JAVA_TOOL_OPTIONS='"-XX:-HeapDumpOnOutOfMemoryError" "-Dmarker=-XX:ErrorFile=/nope"' \ + STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null -# JAVA_TOOL_OPTIONS and JDK_JAVA_OPTIONS are read before the command line, so -# the launcher must leave out a flag already set there rather than override it. -TOOL_OPTIONS_CAPTURE="${TEMP_DIR}/tool-java-options.args" -CAPTURE_FILE="$TOOL_OPTIONS_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ - JAVA_TOOL_OPTIONS="-XX:ErrorFile=/operator/hs_err.log" \ - JDK_JAVA_OPTIONS="-XX:-HeapDumpOnOutOfMemoryError" \ +assert_effective_flag "$QUOTED_TOOL_CAPTURE" HeapDumpOnOutOfMemoryError '^false$' +assert_effective_flag "$QUOTED_TOOL_CAPTURE" ErrorFile "$ERROR_FILE_PATTERN" + +# JDK_JAVA_OPTIONS, including an @argfile, also overrides the defaults. +ARGFILE="${TEMP_DIR}/jdk-java-options.args" +echo '-XX:ErrorFile=/operator/argfile-hs_err.log' > "$ARGFILE" +ARGFILE_CAPTURE="${TEMP_DIR}/argfile.args" +CAPTURE_FILE="$ARGFILE_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + JDK_JAVA_OPTIONS="@${ARGFILE}" \ STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null -assert_no_argument '^-XX:ErrorFile=' "$TOOL_OPTIONS_CAPTURE" -assert_no_argument '^-XX:[+-]HeapDumpOnOutOfMemoryError$' "$TOOL_OPTIONS_CAPTURE" -assert_argument_matching "^-XX:HeapDumpPath=${LOGS_PATTERN}/" "$TOOL_OPTIONS_CAPTURE" +assert_effective_flag "$ARGFILE_CAPTURE" ErrorFile '^/operator/argfile-hs_err\.log$' \ + "@${ARGFILE}" +assert_effective_flag "$ARGFILE_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" "@${ARGFILE}" + +# A launch never reuses a name already taken in logs/, even within the same +# second: with a fixed clock, the counter moves past both existing files. +MOCK_DATE_BIN="${TEMP_DIR}/mock-date-bin" +mkdir -p "$MOCK_DATE_BIN" +printf '#!/bin/bash\necho 20200101-000000\n' > "${MOCK_DATE_BIN}/date" +chmod +x "${MOCK_DATE_BIN}/date" +mkdir -p "${SERVER_ROOT}/logs" +CRASH_NAME_FIXTURES=("${SERVER_ROOT}/logs/java_pid1_20200101-000000.hprof" + "${SERVER_ROOT}/logs/hs_err_pid1_20200101-000000-1.log") +touch "${CRASH_NAME_FIXTURES[@]}" +UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" +CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null + +assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ + "^${LOGS_PATTERN}/java_pid[0-9]+_20200101-000000-2\.hprof$" +assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ + "^${LOGS_PATTERN}/hs_err_pid%p_20200101-000000-2\.log$" +rm -f "${CRASH_NAME_FIXTURES[@]}" +CRASH_NAME_FIXTURES=() -# With telemetry on, the launcher adds its agent to JAVA_TOOL_OPTIONS. It must -# keep the operator's flags there, or the -XX:ErrorFile left out above is lost. +# With telemetry on, the launcher appends its agent to JAVA_TOOL_OPTIONS and +# keeps both the defaults and the operator's flags there. OT_EXPECTED_MD5=$(grep -E '^ *expected_md5=' "$SERVER_SCRIPT" | cut -d'"' -f2) MOCK_MD5_BIN="${TEMP_DIR}/mock-md5-bin" mkdir -p "$MOCK_MD5_BIN" printf '#!/bin/bash\necho "%s $1"\n' "$OT_EXPECTED_MD5" > "${MOCK_MD5_BIN}/md5sum" chmod +x "${MOCK_MD5_BIN}/md5sum" OT_JAR_PATH="${SERVER_ROOT}/plugins/opentelemetry-javaagent.jar" -OT_JAR_EXISTED="false" -if [[ -e "$OT_JAR_PATH" ]]; then - OT_JAR_EXISTED="true" -else +if [[ ! -e "$OT_JAR_PATH" ]]; then mkdir -p "${SERVER_ROOT}/plugins" - : > "$OT_JAR_PATH" + OT_FIXTURE_JAR="$OT_JAR_PATH" + : > "$OT_FIXTURE_JAR" fi TELEMETRY_CAPTURE="${TEMP_DIR}/telemetry.args" -TELEMETRY_TOOL_OPTIONS="${TEMP_DIR}/telemetry-tool-options.txt" -CAPTURE_FILE="$TELEMETRY_CAPTURE" TOOL_OPTIONS_CAPTURE_FILE="$TELEMETRY_TOOL_OPTIONS" \ - JAVA_HOME="$MOCK_JAVA_HOME" PATH="${MOCK_MD5_BIN}:${PATH}" \ +CAPTURE_FILE="$TELEMETRY_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + PATH="${MOCK_MD5_BIN}:${PATH}" \ JAVA_TOOL_OPTIONS="-XX:ErrorFile=/operator/hs_err.log" \ STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true "" "" true \ >/dev/null -if [[ "$OT_JAR_EXISTED" == "false" ]]; then - rm -f "$OT_JAR_PATH" +if [[ -n "${OT_FIXTURE_JAR:-}" ]]; then + rm -f "$OT_FIXTURE_JAR" + OT_FIXTURE_JAR="" fi -assert_no_argument '^-XX:ErrorFile=' "$TELEMETRY_CAPTURE" -grep -Eq -- '(^| )-XX:ErrorFile=/operator/hs_err\.log( |$)' "$TELEMETRY_TOOL_OPTIONS" || +TELEMETRY_TOOL_OPTIONS=$(cat "${TELEMETRY_CAPTURE}.tool-options") +[[ "$TELEMETRY_TOOL_OPTIONS" == -XX:+HeapDumpOnOutOfMemoryError\ * ]] || + fail "telemetry agent setup dropped the crash-file defaults" +[[ "$TELEMETRY_TOOL_OPTIONS" == *" -XX:ErrorFile=/operator/hs_err.log "* ]] || fail "telemetry agent setup dropped the operator's JAVA_TOOL_OPTIONS" -grep -Eq -- '(^| )-javaagent:[^ ]*/opentelemetry-javaagent\.jar$' "$TELEMETRY_TOOL_OPTIONS" || +[[ "$TELEMETRY_TOOL_OPTIONS" =~ \ -javaagent:[^\ ]*/opentelemetry-javaagent\.jar$ ]] || fail "telemetry agent was not added to JAVA_TOOL_OPTIONS" JDK21_CAPTURE="${TEMP_DIR}/jdk21.args" From e7c16d249722ff65dbda66302e3dd627f9e95ff1 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sun, 4 Oct 2026 09:39:17 +0530 Subject: [PATCH 04/23] test(server): keep existing crash files when testing name collisions The collision check created two fixed crash-file names under the supplied distribution's logs/ and removed them afterwards, which would delete real files that already had those names. Record and remove only the files this run creates. --- .../travis/test-java-security-properties.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 772d7864cf..7cbce9940e 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -541,9 +541,15 @@ mkdir -p "$MOCK_DATE_BIN" printf '#!/bin/bash\necho 20200101-000000\n' > "${MOCK_DATE_BIN}/date" chmod +x "${MOCK_DATE_BIN}/date" mkdir -p "${SERVER_ROOT}/logs" -CRASH_NAME_FIXTURES=("${SERVER_ROOT}/logs/java_pid1_20200101-000000.hprof" - "${SERVER_ROOT}/logs/hs_err_pid1_20200101-000000-1.log") -touch "${CRASH_NAME_FIXTURES[@]}" +# Only files this run creates are recorded for removal, so an existing file +# with the same name in the supplied distribution is left alone. +for CRASH_NAME_FIXTURE in "${SERVER_ROOT}/logs/java_pid1_20200101-000000.hprof" \ + "${SERVER_ROOT}/logs/hs_err_pid1_20200101-000000-1.log"; do + if [[ ! -e "$CRASH_NAME_FIXTURE" ]]; then + CRASH_NAME_FIXTURES+=("$CRASH_NAME_FIXTURE") + : > "$CRASH_NAME_FIXTURE" + fi +done UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ @@ -553,7 +559,9 @@ assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ "^${LOGS_PATTERN}/java_pid[0-9]+_20200101-000000-2\.hprof$" assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ "^${LOGS_PATTERN}/hs_err_pid%p_20200101-000000-2\.log$" -rm -f "${CRASH_NAME_FIXTURES[@]}" +if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then + rm -f "${CRASH_NAME_FIXTURES[@]}" +fi CRASH_NAME_FIXTURES=() # With telemetry on, the launcher appends its agent to JAVA_TOOL_OPTIONS and From c29edae13808c25f21cbbd6851ecc296a86ca83b Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sun, 4 Oct 2026 20:55:27 +0530 Subject: [PATCH 05/23] docs(docker): warn that repeated OOM dumps accumulate in the log volume Each launch writes its heap dump to a new file, so a Server that keeps running out of memory adds a heap-sized dump per restart until a persistent /hugegraph-server/logs volume is full. Say that nothing removes old dumps, and how to size the volume, clean up, or turn heap dumps off. --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index e983ce31ee..4402b9508d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,7 +234,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout; their INFO output, the audit log and the slow-query log stay in files only. When a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid_.log`) and out-of-memory heap dumps (`java_pid_.hprof`). `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap; size the volume for it, or pass `-XX:-HeapDumpOnOutOfMemoryError` in `JAVA_OPTS` to turn heap dumps off. +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid_.log`) and out-of-memory heap dumps (`java_pid_.hprof`). `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `java_pid*.hprof` files after each incident, or pass `-XX:-HeapDumpOnOutOfMemoryError` in `JAVA_OPTS` to turn heap dumps off. ## Developers From 7e0772579cdb6c51ed9ef184ec0876aab7e016c1 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Mon, 5 Oct 2026 19:46:54 +0530 Subject: [PATCH 06/23] fix(server): add the host to crash-file names and echo launcher errors - Crash-file names now carry the host name, the pod name on Kubernetes, before the launch time. Pods sharing one log volume each run as PID 1 and could start in the same second; with only the PID and time they could pick the same name. - Launcher errors that stop the Server before Java starts (unsupported JDK, too little free memory, unknown GC option, missing security properties, telemetry agent download or checksum failure) now go to stderr as well as hugegraph-server.log, through one report_error helper. start-hugegraph.sh passes that stderr to the terminal or the container log, so kubectl logs --previous shows these causes too. - Tests cover the host part of the names, another host in the same second, unsafe host characters, and a preflight error on stderr and in the server log. docker/README.md is updated to match. --- docker/README.md | 4 +- .../assembly/static/bin/hugegraph-server.sh | 39 ++++++++------- .../travis/test-java-security-properties.sh | 48 ++++++++++++++----- 3 files changed, 61 insertions(+), 30 deletions(-) diff --git a/docker/README.md b/docker/README.md index 4402b9508d..1c1ae3b0e5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -232,9 +232,9 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t ### Server logs and crash files -Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout; their INFO output, the audit log and the slow-query log stay in files only. When a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. +Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout; their INFO output, the audit log and the slow-query log stay in files only. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid_.log`) and out-of-memory heap dumps (`java_pid_.hprof`). `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `java_pid*.hprof` files after each incident, or pass `-XX:-HeapDumpOnOutOfMemoryError` in `JAVA_OPTS` to turn heap dumps off. +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`java_pid__.hprof`). The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `java_pid*.hprof` files after each incident, or pass `-XX:-HeapDumpOnOutOfMemoryError` in `JAVA_OPTS` to turn heap dumps off. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 4e9ad282ef..2105812b5b 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -58,6 +58,13 @@ GC_OPTION="${SERVER_ARGS[4]:-}" OPEN_TELEMETRY="${SERVER_ARGS[5]:-}" ensure_path_writable "$LOGS" + +# Fatal launcher errors go to stderr, which reaches the terminal or the container +# log, and to ${OUTPUT}, the log start-hugegraph.sh points operators at. +report_error() { + echo "$1" >&2 + echo "$1" >> "${OUTPUT}" +} ensure_path_writable "$PLUGINS" # The maximum and minimum heap memory that service can use @@ -109,8 +116,7 @@ JAVA_VERSION=$($JAVA -version 2>&1 | # Drop any pre-release suffix, e.g. "24-ea" -> "24" JAVA_VERSION="${JAVA_VERSION%%[!0-9]*}" if [[ -z $JAVA_VERSION || $JAVA_VERSION -lt $MIN_JAVA_VERSION ]]; then - echo "Make sure the JDK is installed and the version >= $MIN_JAVA_VERSION, current is $JAVA_VERSION" \ - >> "${OUTPUT}" + report_error "Make sure the JDK is installed and the version >= $MIN_JAVA_VERSION, current is $JAVA_VERSION" exit 1 fi @@ -118,7 +124,7 @@ fi if [ "$JAVA_OPTIONS" = "" ]; then XMX=$(calc_xmx $MIN_MEM $MAX_MEM) if [ $? -ne 0 ]; then - echo "Failed to start HugeGraphServer, requires at least ${MIN_MEM}MB free memory" >> "${OUTPUT}" + report_error "Failed to start HugeGraphServer, requires at least ${MIN_MEM}MB free memory" exit 1 fi JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m ${USER_OPTION}" @@ -134,9 +140,10 @@ fi # and _JAVA_OPTIONS, so any of those overrides them. The JVM does that parsing # itself, including quoted options and @argfiles. # A restarted container often reuses the JVM's PID, and HotSpot will not overwrite -# an existing crash log or heap dump, so the file names carry the launch time plus -# a counter that skips names already used in $LOGS. The script execs java below, -# so $$ is the JVM's PID. +# an existing crash log or heap dump, so the file names carry the host name (the +# pod name on Kubernetes, so pods sharing one log volume do not collide), the +# launch time, and a counter that skips names already used in $LOGS. The script +# execs java below, so $$ is the JVM's PID. crash_files_exist() { local file for file in "${LOGS}"/java_pid*_"$1".hprof "${LOGS}"/hs_err_pid*_"$1".log; do @@ -144,7 +151,8 @@ crash_files_exist() { done return 1 } -LAUNCH_STAMP=$(date +%Y%m%d-%H%M%S) +LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | tr -c 'A-Za-z0-9._-' '_') +LAUNCH_STAMP="${LAUNCH_HOST}_$(date +%Y%m%d-%H%M%S)" LAUNCH_ID="${LAUNCH_STAMP}" LAUNCH_SUFFIX=0 while crash_files_exist "${LAUNCH_ID}"; do @@ -179,7 +187,7 @@ case "$GC_OPTION" in -XX:+UnlockDiagnosticVMOptions -XX:-ZProactive" ;; *) - echo "Unrecognized gc option: '$GC_OPTION', default use g1, options only support 'ZGC' now" >> ${OUTPUT} + report_error "Unrecognized gc option: '$GC_OPTION', default use g1, options only support 'ZGC' now" exit 1 esac @@ -194,8 +202,7 @@ Run the server on Java ${MAX_SECURITY_JAVA_VERSION} or lower, or start it with t disabled: 'start-hugegraph.sh -s false'. EOF ) - echo "${SECURITY_UNSUPPORTED_MSG}" >&2 - echo "${SECURITY_UNSUPPORTED_MSG}" >> "${OUTPUT}" + report_error "${SECURITY_UNSUPPORTED_MSG}" exit 1 fi @@ -223,12 +230,10 @@ EOF # The bootstrap validates the effective policy and refuses to start, but # its stderr goes to the stdout log in daemon mode. Name the cause here # so it also reaches the log start-hugegraph.sh points operators at. - cat >> "${OUTPUT}" < setting a finite positive -networkaddress.cache.ttl. -EOF +networkaddress.cache.ttl." fi JVM_OPTIONS="${JVM_OPTIONS} \ -Djava.security.properties=${SECURITY_PROPERTIES}" @@ -248,7 +253,7 @@ if [ "${OPEN_TELEMETRY}" == "true" ]; then "${GITHUB}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.1.0/${OT_JAR}" if [[ ! -e "${OT_JAR_PATH}" ]]; then - echo "## Error: Failed to download ${OT_JAR}." >>${OUTPUT} + report_error "## Error: Failed to download ${OT_JAR}." exit 1 fi fi @@ -258,8 +263,8 @@ if [ "${OPEN_TELEMETRY}" == "true" ]; then actual_md5=$(md5sum "${OT_JAR_PATH}" | awk '{print $1}') if [[ "${expected_md5}" != "${actual_md5}" ]]; then - echo "## Error: MD5 checksum verification failed for ${OT_JAR_PATH}." >>${OUTPUT} - echo "## Tips: Remove the file and try again." >>${OUTPUT} + report_error "## Error: MD5 checksum verification failed for ${OT_JAR_PATH}." + report_error "## Tips: Remove the file and try again." exit 1 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 7cbce9940e..24c30238e4 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -484,11 +484,12 @@ assert_effective_flag() { } LOGS_PATTERN=$(printf '%s' "${SERVER_ROOT}/logs" | sed 's/[][\.*^$+?(){}|]/\\&/g') -# The names carry the launch time and, if needed, a counter, since a restarted -# container often reuses the PID and HotSpot will not overwrite an existing -# crash log or heap dump. -HEAP_DUMP_PATTERN="^${LOGS_PATTERN}/java_pid[0-9]+_[0-9]{8}-[0-9]{6}(-[0-9]+)?\.hprof$" -ERROR_FILE_PATTERN="^${LOGS_PATTERN}/hs_err_pid%p_[0-9]{8}-[0-9]{6}(-[0-9]+)?\.log$" +# The names carry the host name, the launch time and, if needed, a counter, +# since a restarted container often reuses the PID, pods may share one log +# volume, and HotSpot will not overwrite an existing crash log or heap dump. +NAME_ID_PATTERN="[A-Za-z0-9._-]+_[0-9]{8}-[0-9]{6}(-[0-9]+)?" +HEAP_DUMP_PATTERN="^${LOGS_PATTERN}/java_pid[0-9]+_${NAME_ID_PATTERN}\.hprof$" +ERROR_FILE_PATTERN="^${LOGS_PATTERN}/hs_err_pid%p_${NAME_ID_PATTERN}\.log$" assert_no_argument '^-XX:([+-]HeapDumpOnOutOfMemoryError|HeapDumpPath=|ErrorFile=)' \ "$ENABLED_CAPTURE" @@ -535,7 +536,8 @@ assert_effective_flag "$ARGFILE_CAPTURE" ErrorFile '^/operator/argfile-hs_err\.l assert_effective_flag "$ARGFILE_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" "@${ARGFILE}" # A launch never reuses a name already taken in logs/, even within the same -# second: with a fixed clock, the counter moves past both existing files. +# second: with a fixed clock and host name, the counter moves past both +# existing files. MOCK_DATE_BIN="${TEMP_DIR}/mock-date-bin" mkdir -p "$MOCK_DATE_BIN" printf '#!/bin/bash\necho 20200101-000000\n' > "${MOCK_DATE_BIN}/date" @@ -543,8 +545,8 @@ chmod +x "${MOCK_DATE_BIN}/date" mkdir -p "${SERVER_ROOT}/logs" # Only files this run creates are recorded for removal, so an existing file # with the same name in the supplied distribution is left alone. -for CRASH_NAME_FIXTURE in "${SERVER_ROOT}/logs/java_pid1_20200101-000000.hprof" \ - "${SERVER_ROOT}/logs/hs_err_pid1_20200101-000000-1.log"; do +for CRASH_NAME_FIXTURE in "${SERVER_ROOT}/logs/java_pid1_test-pod-a_20200101-000000.hprof" \ + "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log"; do if [[ ! -e "$CRASH_NAME_FIXTURE" ]]; then CRASH_NAME_FIXTURES+=("$CRASH_NAME_FIXTURE") : > "$CRASH_NAME_FIXTURE" @@ -552,18 +554,42 @@ for CRASH_NAME_FIXTURE in "${SERVER_ROOT}/logs/java_pid1_20200101-000000.hprof" done UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ - PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ + HOSTNAME=test-pod-a PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ - "^${LOGS_PATTERN}/java_pid[0-9]+_20200101-000000-2\.hprof$" + "^${LOGS_PATTERN}/java_pid[0-9]+_test-pod-a_20200101-000000-2\.hprof$" assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ - "^${LOGS_PATTERN}/hs_err_pid%p_20200101-000000-2\.log$" + "^${LOGS_PATTERN}/hs_err_pid%p_test-pod-a_20200101-000000-2\.log$" + +# Another pod sharing the volume gets its own names in the same second, and +# characters unsafe in a file name are replaced. +OTHER_HOST_CAPTURE="${TEMP_DIR}/other-host.args" +CAPTURE_FILE="$OTHER_HOST_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + HOSTNAME='test/pod-b' PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null + +assert_effective_flag "$OTHER_HOST_CAPTURE" HeapDumpPath \ + "^${LOGS_PATTERN}/java_pid[0-9]+_test_pod-b_20200101-000000\.hprof$" if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then rm -f "${CRASH_NAME_FIXTURES[@]}" fi CRASH_NAME_FIXTURES=() +# A launcher error before Java starts must reach stderr, which start-hugegraph.sh +# passes to the terminal or container log, as well as the server log. +: > "$SERVER_LOG" +PREFLIGHT_ERROR="${TEMP_DIR}/preflight.err" +if JAVA_HOME="$MOCK_JAVA_HOME" STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true "" "bad-gc" \ + >/dev/null 2>"$PREFLIGHT_ERROR"; then + fail "launcher accepted an unknown GC option" +fi +grep -Fq "Unrecognized gc option: 'bad-gc'" "$PREFLIGHT_ERROR" || + fail "launcher preflight error did not reach stderr" +grep -Fq "Unrecognized gc option: 'bad-gc'" "$SERVER_LOG" || + fail "launcher preflight error did not reach the server log" + # With telemetry on, the launcher appends its agent to JAVA_TOOL_OPTIONS and # keeps both the defaults and the operator's flags there. OT_EXPECTED_MD5=$(grep -E '^ *expected_md5=' "$SERVER_SCRIPT" | cut -d'"' -f2) From ce8cab86f756088bcdccb45f688d8e01b172989e Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Tue, 6 Oct 2026 00:52:10 +0530 Subject: [PATCH 07/23] docs(docker): say how to turn off heap dumps without losing JAVA_OPTS Setting JAVA_OPTS replaces the image default (UseContainerSupport, MaxRAMPercentage=50, ...). Recommend JAVA_TOOL_OPTIONS for the -XX:-HeapDumpOnOutOfMemoryError opt-out, which the launcher reads after its own defaults and which keeps JAVA_OPTS intact, and note the caveat for JAVA_OPTS. --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 1c1ae3b0e5..21de20e780 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,7 +234,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout; their INFO output, the audit log and the slow-query log stay in files only. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`java_pid__.hprof`). The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `java_pid*.hprof` files after each incident, or pass `-XX:-HeapDumpOnOutOfMemoryError` in `JAVA_OPTS` to turn heap dumps off. +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`java_pid__.hprof`). The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `java_pid*.hprof` files after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. ## Developers From 0dd3fef6a4ba558b23f0710cc03a34caee511ddb Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Tue, 6 Oct 2026 09:19:34 +0530 Subject: [PATCH 08/23] docs(docker): only INFO-level library loggers keep INFO in files The ZooKeeper and SOFA loggers are set to WARN, so their INFO events are discarded, not kept in files. Limit the file-only INFO statement to the Hadoop, Netty and Commons loggers. --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 21de20e780..9106a43b34 100644 --- a/docker/README.md +++ b/docker/README.md @@ -232,7 +232,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t ### Server logs and crash files -Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout; their INFO output, the audit log and the slow-query log stay in files only. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. +Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`java_pid__.hprof`). The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `java_pid*.hprof` files after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. From 640f971bfccd96c586179d5bfa468fb0c26f0889 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 6 Oct 2026 19:13:48 +0800 Subject: [PATCH 09/23] fix(server): report unwritable startup paths --- .../src/assembly/static/bin/hugegraph-server.sh | 13 ++++++++----- .../hugegraph-dist/src/assembly/static/bin/util.sh | 8 +++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 74fc301077..f35c2a3337 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -57,15 +57,18 @@ USER_OPTION="${SERVER_ARGS[3]:-}" GC_OPTION="${SERVER_ARGS[4]:-}" OPEN_TELEMETRY="${SERVER_ARGS[5]:-}" -ensure_path_writable "$LOGS" - # Fatal launcher errors go to stderr, which reaches the terminal or the container -# log, and to ${OUTPUT}, the log start-hugegraph.sh points operators at. +# log, and, when possible, to ${OUTPUT}, the log start-hugegraph.sh points +# operators at. report_error() { echo "$1" >&2 - echo "$1" >> "${OUTPUT}" + if [[ -w "${LOGS}" ]]; then + { echo "$1" >> "${OUTPUT}"; } 2>/dev/null || true + fi } -ensure_path_writable "$PLUGINS" + +ensure_path_writable "$LOGS" report_error +ensure_path_writable "$PLUGINS" report_error # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index 8ae46e88a4..fb8e701bcc 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -473,13 +473,19 @@ function remove_with_prompt() { function ensure_path_writable() { local path=$1 + local error_handler="${2:-}" # Ensure input path exist if [ ! -d "${path}" ]; then mkdir -p "${path}" fi # Check for write permission if [ ! -w "${path}" ]; then - echo "No write permission on directory ${path}" + local message="No write permission on directory ${path}" + if [ -n "${error_handler}" ]; then + "${error_handler}" "${message}" + else + echo "${message}" + fi exit 1 fi } From c4e481f3d123d960988f4afd1c096aee6ab6f6dd Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Tue, 6 Oct 2026 23:46:35 +0530 Subject: [PATCH 10/23] fix(server): give every JVM its own heap dump file Child JVMs the Server starts, such as Hadoop computer jobs, inherit JAVA_TOOL_OPTIONS. HeapDumpPath does not expand %p, so the fixed file path built from the launcher's PID was shared: if a child dumped first, HotSpot would not overwrite the file on a later Server OOM. HeapDumpPath now names a per-launch directory, logs/heapdump__/, in which HotSpot writes java_pid.hprof for each JVM. ErrorFile already expands %p. Because the directory exists even when nothing is dumped, each start removes this host's empty heapdump_* directories from earlier launches; rmdir never removes a directory holding a dump. The test checks the directory exists, runs two real JVMs out of memory with the launcher's options and expects one dump each, and covers the name counter and the empty-directory cleanup. --- docker/README.md | 2 +- .../assembly/static/bin/hugegraph-server.sh | 28 ++++-- .../travis/test-java-security-properties.sh | 86 ++++++++++++++++--- 3 files changed, 94 insertions(+), 22 deletions(-) diff --git a/docker/README.md b/docker/README.md index 9106a43b34..f9602a9b5a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,7 +234,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`java_pid__.hprof`). The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `java_pid*.hprof` files after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC; at startup it only removes this host's empty `heapdump_*` directories from earlier launches. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index f35c2a3337..44ea471e4c 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -148,19 +148,28 @@ fi # own JAVA_TOOL_OPTIONS, JDK_JAVA_OPTIONS, the command line (JAVA_OPTIONS and -j) # and _JAVA_OPTIONS, so any of those overrides them. The JVM does that parsing # itself, including quoted options and @argfiles. -# A restarted container often reuses the JVM's PID, and HotSpot will not overwrite -# an existing crash log or heap dump, so the file names carry the host name (the -# pod name on Kubernetes, so pods sharing one log volume do not collide), the -# launch time, and a counter that skips names already used in $LOGS. The script -# execs java below, so $$ is the JVM's PID. +# Child JVMs the Server starts (computer jobs, for example) inherit these options, +# so every path must stay unique per JVM. ErrorFile expands %p to each JVM's PID. +# HeapDumpPath does not expand %p, but when it names an existing directory each JVM +# writes java_pid.hprof inside it, so it points at a directory per launch. +# A restarted container often reuses the PID, and HotSpot will not overwrite an +# existing crash log or heap dump, so the names carry the host name (the pod name +# on Kubernetes, so pods sharing one log volume do not collide), the launch time, +# and a counter that skips names already used in $LOGS. crash_files_exist() { + [[ -e "${LOGS}/heapdump_$1" ]] && return 0 local file - for file in "${LOGS}"/java_pid*_"$1".hprof "${LOGS}"/hs_err_pid*_"$1".log; do + for file in "${LOGS}"/hs_err_pid*_"$1".log; do [[ -e ${file} ]] && return 0 done return 1 } LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | tr -c 'A-Za-z0-9._-' '_') +# Earlier launches on this host leave their heap dump directory behind even when +# nothing was dumped. Remove those that are empty; rmdir never removes a dump. +for HEAP_DUMP_DIR in "${LOGS}"/heapdump_"${LAUNCH_HOST}"_*/; do + [[ -d ${HEAP_DUMP_DIR} ]] && rmdir "${HEAP_DUMP_DIR}" 2>/dev/null +done LAUNCH_STAMP="${LAUNCH_HOST}_$(date +%Y%m%d-%H%M%S)" LAUNCH_ID="${LAUNCH_STAMP}" LAUNCH_SUFFIX=0 @@ -168,8 +177,13 @@ while crash_files_exist "${LAUNCH_ID}"; do LAUNCH_SUFFIX=$((LAUNCH_SUFFIX + 1)) LAUNCH_ID="${LAUNCH_STAMP}-${LAUNCH_SUFFIX}" done +HEAP_DUMP_DIR="${LOGS}/heapdump_${LAUNCH_ID}" +mkdir -p "${HEAP_DUMP_DIR}" || { + report_error "Failed to create heap dump directory ${HEAP_DUMP_DIR}" + exit 1 +} CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" -CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${LOGS}/java_pid$$_${LAUNCH_ID}.hprof\"" +CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${HEAP_DUMP_DIR}\"" CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_ID}.log\"" export JAVA_TOOL_OPTIONS="${CRASH_OPTIONS}${JAVA_TOOL_OPTIONS:+ ${JAVA_TOOL_OPTIONS}}" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 93ac04ecc4..9be9de2b2c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -224,8 +224,15 @@ cleanup() { rm -f "$OT_FIXTURE_JAR" fi if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then - rm -f "${CRASH_NAME_FIXTURES[@]}" + rm -rf "${CRASH_NAME_FIXTURES[@]}" fi + # Each launcher run creates a heap dump directory; drop the empty ones. + local dump_dir + for dump_dir in "${SERVER_ROOT}"/logs/heapdump_*/; do + if [[ -d "$dump_dir" ]]; then + rmdir "$dump_dir" 2>/dev/null || true + fi + done rm -rf "$TEMP_DIR" } @@ -614,8 +621,10 @@ LOGS_PATTERN=$(printf '%s' "${SERVER_ROOT}/logs" | sed 's/[][\.*^$+?(){}|]/\\&/g # The names carry the host name, the launch time and, if needed, a counter, # since a restarted container often reuses the PID, pods may share one log # volume, and HotSpot will not overwrite an existing crash log or heap dump. +# HeapDumpPath names a directory per launch, so each JVM writes its own +# java_pid.hprof there. NAME_ID_PATTERN="[A-Za-z0-9._-]+_[0-9]{8}-[0-9]{6}(-[0-9]+)?" -HEAP_DUMP_PATTERN="^${LOGS_PATTERN}/java_pid[0-9]+_${NAME_ID_PATTERN}\.hprof$" +HEAP_DUMP_PATTERN="^${LOGS_PATTERN}/heapdump_${NAME_ID_PATTERN}$" ERROR_FILE_PATTERN="^${LOGS_PATTERN}/hs_err_pid%p_${NAME_ID_PATTERN}\.log$" assert_no_argument '^-XX:([+-]HeapDumpOnOutOfMemoryError|HeapDumpPath=|ErrorFile=)' \ @@ -623,6 +632,36 @@ assert_no_argument '^-XX:([+-]HeapDumpOnOutOfMemoryError|HeapDumpPath=|ErrorFile assert_effective_flag "$ENABLED_CAPTURE" HeapDumpOnOutOfMemoryError '^true$' assert_effective_flag "$ENABLED_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" assert_effective_flag "$ENABLED_CAPTURE" ErrorFile "$ERROR_FILE_PATTERN" +ENABLED_DUMP_DIR=$(effective_flag "$ENABLED_CAPTURE" HeapDumpPath) +[[ -d "$ENABLED_DUMP_DIR" ]] || + fail "launcher did not create the heap dump directory ${ENABLED_DUMP_DIR}" + +# Child JVMs the Server starts inherit JAVA_TOOL_OPTIONS. Two JVMs started with +# the launcher's options, like the Server and a computer job, must each write +# their own heap dump into the launch directory instead of competing for one. +OOM_SOURCE="${TEMP_DIR}/OomCheck.java" +cat > "$OOM_SOURCE" <<'JAVA' +import java.util.ArrayList; +import java.util.List; + +public class OomCheck { + public static void main(String[] args) { + List hold = new ArrayList<>(); + while (true) { + hold.add(new long[1 << 20]); + } + } +} +JAVA +for OOM_RUN in 1 2; do + JAVA_TOOL_OPTIONS="$(cat "${ENABLED_CAPTURE}.tool-options")" \ + "$JAVA_BIN" -Xmx64m "$OOM_SOURCE" >/dev/null 2>&1 || true +done +OOM_DUMPS=$(find "$ENABLED_DUMP_DIR" -maxdepth 1 -name 'java_pid*.hprof' | wc -l) +rm -f "$ENABLED_DUMP_DIR"/java_pid*.hprof +if [[ "$OOM_DUMPS" -ne 2 ]]; then + fail "expected a heap dump per JVM in ${ENABLED_DUMP_DIR}, found ${OOM_DUMPS}" +fi # JAVA_OPTIONS replaces the default heap options; the crash defaults stay, and # paths given there win. @@ -663,31 +702,50 @@ assert_effective_flag "$ARGFILE_CAPTURE" ErrorFile '^/operator/argfile-hs_err\.l assert_effective_flag "$ARGFILE_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" "@${ARGFILE}" # A launch never reuses a name already taken in logs/, even within the same -# second: with a fixed clock and host name, the counter moves past both -# existing files. +# second: with a fixed clock and host name, the counter moves past an existing +# heap dump directory and crash log. Empty dump directories from this host's +# earlier launches are removed; other hosts' and non-empty ones stay. MOCK_DATE_BIN="${TEMP_DIR}/mock-date-bin" mkdir -p "$MOCK_DATE_BIN" printf '#!/bin/bash\necho 20200101-000000\n' > "${MOCK_DATE_BIN}/date" chmod +x "${MOCK_DATE_BIN}/date" mkdir -p "${SERVER_ROOT}/logs" -# Only files this run creates are recorded for removal, so an existing file +# Only fixtures this run creates are recorded for removal, so an existing file # with the same name in the supplied distribution is left alone. -for CRASH_NAME_FIXTURE in "${SERVER_ROOT}/logs/java_pid1_test-pod-a_20200101-000000.hprof" \ - "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log"; do - if [[ ! -e "$CRASH_NAME_FIXTURE" ]]; then - CRASH_NAME_FIXTURES+=("$CRASH_NAME_FIXTURE") - : > "$CRASH_NAME_FIXTURE" +add_crash_fixture() { + local path="$1" + local kind="$2" + if [[ ! -e "$path" ]]; then + CRASH_NAME_FIXTURES+=("$path") + case "$kind" in + file) : > "$path" ;; + empty-dir) mkdir "$path" ;; + dump-dir) mkdir "$path" && : > "${path}/java_pid1.hprof" ;; + esac fi -done +} +USED_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000" +OLD_EMPTY_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000000" +OTHER_HOST_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-c_20191231-000000" +add_crash_fixture "$USED_DUMP_DIR" dump-dir +add_crash_fixture "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log" file +add_crash_fixture "$OLD_EMPTY_DUMP_DIR" empty-dir +add_crash_fixture "$OTHER_HOST_DUMP_DIR" empty-dir UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ HOSTNAME=test-pod-a PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ - "^${LOGS_PATTERN}/java_pid[0-9]+_test-pod-a_20200101-000000-2\.hprof$" + "^${LOGS_PATTERN}/heapdump_test-pod-a_20200101-000000-2$" assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ "^${LOGS_PATTERN}/hs_err_pid%p_test-pod-a_20200101-000000-2\.log$" +[[ ! -e "$OLD_EMPTY_DUMP_DIR" ]] || + fail "launcher kept an empty heap dump directory from an earlier launch" +[[ -d "$OTHER_HOST_DUMP_DIR" ]] || + fail "launcher removed another host's heap dump directory" +[[ -f "${USED_DUMP_DIR}/java_pid1.hprof" ]] || + fail "launcher removed an existing heap dump" # Another pod sharing the volume gets its own names in the same second, and # characters unsafe in a file name are replaced. @@ -697,9 +755,9 @@ CAPTURE_FILE="$OTHER_HOST_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null assert_effective_flag "$OTHER_HOST_CAPTURE" HeapDumpPath \ - "^${LOGS_PATTERN}/java_pid[0-9]+_test_pod-b_20200101-000000\.hprof$" + "^${LOGS_PATTERN}/heapdump_test_pod-b_20200101-000000$" if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then - rm -f "${CRASH_NAME_FIXTURES[@]}" + rm -rf "${CRASH_NAME_FIXTURES[@]}" fi CRASH_NAME_FIXTURES=() From ac68cf32b6993bc15fba92b7c618eedbf099c2fb Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Wed, 7 Oct 2026 09:49:45 +0530 Subject: [PATCH 11/23] fix(server): only remove heap dump directories whose JVM has exited A live JVM's heap dump directory stays empty until it runs out of memory, so removing every empty heapdump_* directory at startup, or in the test's EXIT cleanup, could take the target from a running Server that shares the logs directory. Each launch now writes /.owner with the JVM's PID and its process start time (the script execs java, so both are the JVM's). Startup removes a directory only when it holds nothing but .owner and no process with that PID and start time exists, so a PID reused after a container restart does not look like the old owner. Directories without a marker, with a dump, or owned by a running process stay, and nothing is removed when ps cannot report a start time. The test's cleanup removes only directories created during the run, and the test covers a dead owner, a live owner, an unowned directory, another host's directory and one holding a dump. --- docker/README.md | 2 +- .../assembly/static/bin/hugegraph-server.sh | 34 +++++++++++--- .../travis/test-java-security-properties.sh | 45 +++++++++++++++---- 3 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docker/README.md b/docker/README.md index f9602a9b5a..03b1e724f8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,7 +234,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC; at startup it only removes this host's empty `heapdump_*` directories from earlier launches. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC; at startup it only removes this host's `heapdump_*` directories that hold no dump and whose JVM has exited (each records its owner in a `.owner` file). Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 44ea471e4c..bd7bef36ba 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -165,11 +165,33 @@ crash_files_exist() { return 1 } LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | tr -c 'A-Za-z0-9._-' '_') -# Earlier launches on this host leave their heap dump directory behind even when -# nothing was dumped. Remove those that are empty; rmdir never removes a dump. -for HEAP_DUMP_DIR in "${LOGS}"/heapdump_"${LAUNCH_HOST}"_*/; do - [[ -d ${HEAP_DUMP_DIR} ]] && rmdir "${HEAP_DUMP_DIR}" 2>/dev/null -done +# Each launch records the JVM that owns its heap dump directory in .owner: the PID +# and that process's start time, so a reused PID (common after a container +# restart) does not look like the old owner. A live JVM's directory stays empty +# until it runs out of memory, so startup removes a directory only when it holds +# nothing but .owner and that exact process is gone. Directories without a marker, +# with a dump, or owned by a running process are left alone, and nothing is +# removed when ps cannot tell. +process_identity() { + # Collapse ps's padding so the value reads back unchanged from .owner. + ps -o lstart= -p "$1" 2>/dev/null | awk '{$1 = $1; print}' +} +remove_unused_heap_dump_dir() { + local dir="$1" owner_pid="" owner_start="" entries + [[ -f "${dir}/.owner" ]] || return 0 + entries=$(ls -A "${dir}") + [[ ${entries} == ".owner" ]] || return 0 + { read -r owner_pid; read -r owner_start; } < "${dir}/.owner" + [[ -n ${owner_pid} && -n ${owner_start} ]] || return 0 + [[ $(process_identity "${owner_pid}") == "${owner_start}" ]] && return 0 + rm -f "${dir}/.owner" && rmdir "${dir}" 2>/dev/null + return 0 +} +if command -v ps >/dev/null 2>&1 && [[ -n $(process_identity $$) ]]; then + for HEAP_DUMP_DIR in "${LOGS}"/heapdump_"${LAUNCH_HOST}"_*/; do + [[ -d ${HEAP_DUMP_DIR} ]] && remove_unused_heap_dump_dir "${HEAP_DUMP_DIR%/}" + done +fi LAUNCH_STAMP="${LAUNCH_HOST}_$(date +%Y%m%d-%H%M%S)" LAUNCH_ID="${LAUNCH_STAMP}" LAUNCH_SUFFIX=0 @@ -182,6 +204,8 @@ mkdir -p "${HEAP_DUMP_DIR}" || { report_error "Failed to create heap dump directory ${HEAP_DUMP_DIR}" exit 1 } +# The script execs java below, so $$ and its start time are the JVM's. +printf '%s\n%s\n' "$$" "$(process_identity $$)" > "${HEAP_DUMP_DIR}/.owner" 2>/dev/null || true CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${HEAP_DUMP_DIR}\"" CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_ID}.log\"" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 9be9de2b2c..34261fb058 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -208,6 +208,9 @@ fi TEMP_DIR=$(mktemp -d) CRASH_NAME_FIXTURES=() OT_FIXTURE_JAR="" +# Heap dump directories that exist before the test belong to someone else (a +# running Server, say), so the cleanup below never touches them. +PRE_EXISTING_DUMP_DIRS=$(ls -d "${SERVER_ROOT}"/logs/heapdump_*/ 2>/dev/null || true) SECURITY_PROPERTIES_BACKUP="${TEMP_DIR}/java-security.properties" cleanup() { @@ -226,10 +229,13 @@ cleanup() { if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then rm -rf "${CRASH_NAME_FIXTURES[@]}" fi - # Each launcher run creates a heap dump directory; drop the empty ones. + # Each launcher run creates a heap dump directory. Remove only those this + # test created and that hold nothing but their owner marker. local dump_dir for dump_dir in "${SERVER_ROOT}"/logs/heapdump_*/; do - if [[ -d "$dump_dir" ]]; then + if [[ -d "$dump_dir" ]] && + ! grep -Fxq -- "$dump_dir" <<< "$PRE_EXISTING_DUMP_DIRS"; then + rm -f "${dump_dir}.owner" rmdir "$dump_dir" 2>/dev/null || true fi done @@ -635,6 +641,8 @@ assert_effective_flag "$ENABLED_CAPTURE" ErrorFile "$ERROR_FILE_PATTERN" ENABLED_DUMP_DIR=$(effective_flag "$ENABLED_CAPTURE" HeapDumpPath) [[ -d "$ENABLED_DUMP_DIR" ]] || fail "launcher did not create the heap dump directory ${ENABLED_DUMP_DIR}" +[[ -f "${ENABLED_DUMP_DIR}/.owner" ]] || + fail "launcher did not record the owner of ${ENABLED_DUMP_DIR}" # Child JVMs the Server starts inherit JAVA_TOOL_OPTIONS. Two JVMs started with # the launcher's options, like the Server and a computer job, must each write @@ -703,8 +711,10 @@ assert_effective_flag "$ARGFILE_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" "@${A # A launch never reuses a name already taken in logs/, even within the same # second: with a fixed clock and host name, the counter moves past an existing -# heap dump directory and crash log. Empty dump directories from this host's -# earlier launches are removed; other hosts' and non-empty ones stay. +# heap dump directory and crash log. Of this host's earlier dump directories, +# only one whose recorded owner process is gone and that holds no dump is +# removed; one owned by a running process, one without an owner marker, +# another host's, and one holding a dump all stay. MOCK_DATE_BIN="${TEMP_DIR}/mock-date-bin" mkdir -p "$MOCK_DATE_BIN" printf '#!/bin/bash\necho 20200101-000000\n' > "${MOCK_DATE_BIN}/date" @@ -721,15 +731,30 @@ add_crash_fixture() { file) : > "$path" ;; empty-dir) mkdir "$path" ;; dump-dir) mkdir "$path" && : > "${path}/java_pid1.hprof" ;; + dead-owner) + # A finished process: its PID no longer has this start time. + sleep 0 & + local dead_pid=$! + wait "$dead_pid" || true + mkdir "$path" && + printf '%s\n%s\n' "$dead_pid" "Thu Jan 1 00:00:00 1970" > "${path}/.owner" ;; + live-owner) + mkdir "$path" && + printf '%s\n%s\n' "$$" "$(ps -o lstart= -p $$ | awk '{$1 = $1; print}')" \ + > "${path}/.owner" ;; esac fi } USED_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000" -OLD_EMPTY_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000000" +DEAD_OWNER_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000000" +LIVE_OWNER_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000001" +UNOWNED_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000002" OTHER_HOST_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-c_20191231-000000" add_crash_fixture "$USED_DUMP_DIR" dump-dir add_crash_fixture "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log" file -add_crash_fixture "$OLD_EMPTY_DUMP_DIR" empty-dir +add_crash_fixture "$DEAD_OWNER_DUMP_DIR" dead-owner +add_crash_fixture "$LIVE_OWNER_DUMP_DIR" live-owner +add_crash_fixture "$UNOWNED_DUMP_DIR" empty-dir add_crash_fixture "$OTHER_HOST_DUMP_DIR" empty-dir UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ @@ -740,8 +765,12 @@ assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ "^${LOGS_PATTERN}/heapdump_test-pod-a_20200101-000000-2$" assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ "^${LOGS_PATTERN}/hs_err_pid%p_test-pod-a_20200101-000000-2\.log$" -[[ ! -e "$OLD_EMPTY_DUMP_DIR" ]] || - fail "launcher kept an empty heap dump directory from an earlier launch" +[[ ! -e "$DEAD_OWNER_DUMP_DIR" ]] || + fail "launcher kept an unused heap dump directory whose owner is gone" +[[ -d "$LIVE_OWNER_DUMP_DIR" ]] || + fail "launcher removed the heap dump directory of a running process" +[[ -d "$UNOWNED_DUMP_DIR" ]] || + fail "launcher removed a heap dump directory without an owner marker" [[ -d "$OTHER_HOST_DUMP_DIR" ]] || fail "launcher removed another host's heap dump directory" [[ -f "${USED_DUMP_DIR}/java_pid1.hprof" ]] || From 4778531c78af676d968eb4ab4d698807f97d6e33 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Wed, 7 Oct 2026 11:12:01 +0530 Subject: [PATCH 12/23] build: exclude everything under logs/ from the RAT license check RAT scans the built distribution under hugegraph-server/ and skipped only **/logs/*.log. Each Server start now leaves a logs/heapdump__/.owner marker, so any CI step or local run that starts the dist before a later Maven build failed the license check with one unapproved file. A heap dump left in logs/ would fail it the same way. Everything under logs/ is runtime output, and no tracked file lives there. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4aac2b9515..d3aadcb2b8 100644 --- a/pom.xml +++ b/pom.xml @@ -253,7 +253,7 @@ scripts/dev/reviewers **/*.md **/dependency-reduced-pom.xml - **/logs/*.log + **/logs/** **/META-INF/**/* style/* ChangeLog From 09a9812ad300dabf00f017bb90c3015db0d0999e Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Wed, 7 Oct 2026 11:32:57 +0530 Subject: [PATCH 13/23] fix(server): name the heap dump owner marker pid, keep pom.xml as is The previous commit widened the RAT exclusion in pom.xml so the .owner marker would pass the license check. The CI Maven cache is keyed on a hash of the pom.xml files, so that change made every job start from an empty cache, where hg-pd-common fails to resolve hugegraph:${revision} during compile. Revert it, leaving pom.xml identical to master. Name the marker pid instead. RAT already excludes files named pid (**/pid), and the file holds the owning JVM's PID and start time. --- docker/README.md | 2 +- .../assembly/static/bin/hugegraph-server.sh | 26 +++++++++---------- .../travis/test-java-security-properties.sh | 8 +++--- pom.xml | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docker/README.md b/docker/README.md index 03b1e724f8..a5941cc411 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,7 +234,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC; at startup it only removes this host's `heapdump_*` directories that hold no dump and whose JVM has exited (each records its owner in a `.owner` file). Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC; at startup it only removes this host's `heapdump_*` directories that hold no dump and whose JVM has exited (each records its owner in a `pid` file). Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index bd7bef36ba..62e20d7afb 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -165,26 +165,26 @@ crash_files_exist() { return 1 } LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | tr -c 'A-Za-z0-9._-' '_') -# Each launch records the JVM that owns its heap dump directory in .owner: the PID -# and that process's start time, so a reused PID (common after a container -# restart) does not look like the old owner. A live JVM's directory stays empty -# until it runs out of memory, so startup removes a directory only when it holds -# nothing but .owner and that exact process is gone. Directories without a marker, -# with a dump, or owned by a running process are left alone, and nothing is -# removed when ps cannot tell. +# Each launch records the JVM that owns its heap dump directory in a file named +# pid, a name the license check already skips: the PID and that process's start +# time, so a reused PID (common after a container restart) does not look like the +# old owner. A live JVM's directory stays empty until it runs out of memory, so +# startup removes a directory only when it holds nothing but that file and that +# exact process is gone. Directories without a marker, with a dump, or owned by a +# running process are left alone, and nothing is removed when ps cannot tell. process_identity() { - # Collapse ps's padding so the value reads back unchanged from .owner. + # Collapse ps's padding so the value reads back unchanged from pid. ps -o lstart= -p "$1" 2>/dev/null | awk '{$1 = $1; print}' } remove_unused_heap_dump_dir() { local dir="$1" owner_pid="" owner_start="" entries - [[ -f "${dir}/.owner" ]] || return 0 + [[ -f "${dir}/pid" ]] || return 0 entries=$(ls -A "${dir}") - [[ ${entries} == ".owner" ]] || return 0 - { read -r owner_pid; read -r owner_start; } < "${dir}/.owner" + [[ ${entries} == "pid" ]] || return 0 + { read -r owner_pid; read -r owner_start; } < "${dir}/pid" [[ -n ${owner_pid} && -n ${owner_start} ]] || return 0 [[ $(process_identity "${owner_pid}") == "${owner_start}" ]] && return 0 - rm -f "${dir}/.owner" && rmdir "${dir}" 2>/dev/null + rm -f "${dir}/pid" && rmdir "${dir}" 2>/dev/null return 0 } if command -v ps >/dev/null 2>&1 && [[ -n $(process_identity $$) ]]; then @@ -205,7 +205,7 @@ mkdir -p "${HEAP_DUMP_DIR}" || { exit 1 } # The script execs java below, so $$ and its start time are the JVM's. -printf '%s\n%s\n' "$$" "$(process_identity $$)" > "${HEAP_DUMP_DIR}/.owner" 2>/dev/null || true +printf '%s\n%s\n' "$$" "$(process_identity $$)" > "${HEAP_DUMP_DIR}/pid" 2>/dev/null || true CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${HEAP_DUMP_DIR}\"" CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_ID}.log\"" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 34261fb058..7882d6067d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -235,7 +235,7 @@ cleanup() { for dump_dir in "${SERVER_ROOT}"/logs/heapdump_*/; do if [[ -d "$dump_dir" ]] && ! grep -Fxq -- "$dump_dir" <<< "$PRE_EXISTING_DUMP_DIRS"; then - rm -f "${dump_dir}.owner" + rm -f "${dump_dir}pid" rmdir "$dump_dir" 2>/dev/null || true fi done @@ -641,7 +641,7 @@ assert_effective_flag "$ENABLED_CAPTURE" ErrorFile "$ERROR_FILE_PATTERN" ENABLED_DUMP_DIR=$(effective_flag "$ENABLED_CAPTURE" HeapDumpPath) [[ -d "$ENABLED_DUMP_DIR" ]] || fail "launcher did not create the heap dump directory ${ENABLED_DUMP_DIR}" -[[ -f "${ENABLED_DUMP_DIR}/.owner" ]] || +[[ -f "${ENABLED_DUMP_DIR}/pid" ]] || fail "launcher did not record the owner of ${ENABLED_DUMP_DIR}" # Child JVMs the Server starts inherit JAVA_TOOL_OPTIONS. Two JVMs started with @@ -737,11 +737,11 @@ add_crash_fixture() { local dead_pid=$! wait "$dead_pid" || true mkdir "$path" && - printf '%s\n%s\n' "$dead_pid" "Thu Jan 1 00:00:00 1970" > "${path}/.owner" ;; + printf '%s\n%s\n' "$dead_pid" "Thu Jan 1 00:00:00 1970" > "${path}/pid" ;; live-owner) mkdir "$path" && printf '%s\n%s\n' "$$" "$(ps -o lstart= -p $$ | awk '{$1 = $1; print}')" \ - > "${path}/.owner" ;; + > "${path}/pid" ;; esac fi } diff --git a/pom.xml b/pom.xml index d3aadcb2b8..4aac2b9515 100644 --- a/pom.xml +++ b/pom.xml @@ -253,7 +253,7 @@ scripts/dev/reviewers **/*.md **/dependency-reduced-pom.xml - **/logs/** + **/logs/*.log **/META-INF/**/* style/* ChangeLog From d591173392a415b1e9d0bef291de8de5279330bd Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Wed, 7 Oct 2026 15:12:49 +0530 Subject: [PATCH 14/23] fix(server): never remove heap dump directories automatically Startup cleanup removed a heapdump_* directory once the Server JVM that created it had exited. Computer-job JVMs inherit the same HeapDumpPath and can outlive their Server, so the next start could remove a directory a running child still needs, and that child's OOM dump would then fail. The launcher has no portable way to see those children. Drop the cleanup and the pid owner marker. Each start leaves one directory, empty unless something ran out of memory, and the README says old ones can be deleted once no HugeGraph JVM from that launch is running. The test checks that an empty directory from an earlier launch is kept. --- docker/README.md | 2 +- .../assembly/static/bin/hugegraph-server.sh | 32 ++------------- .../travis/test-java-security-properties.sh | 41 ++++--------------- 3 files changed, 13 insertions(+), 62 deletions(-) diff --git a/docker/README.md b/docker/README.md index a5941cc411..6a94002166 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,7 +234,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC; at startup it only removes this host's `heapdump_*` directories that hold no dump and whose JVM has exited (each records its owner in a `pid` file). Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. +Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps or `heapdump_*` directories, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Each start leaves a directory behind, empty unless something ran out of memory, because a computer-job JVM can keep using it after its Server exits; old ones can be deleted once no HugeGraph JVM from that launch is running. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 62e20d7afb..cb84251961 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -165,33 +165,9 @@ crash_files_exist() { return 1 } LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | tr -c 'A-Za-z0-9._-' '_') -# Each launch records the JVM that owns its heap dump directory in a file named -# pid, a name the license check already skips: the PID and that process's start -# time, so a reused PID (common after a container restart) does not look like the -# old owner. A live JVM's directory stays empty until it runs out of memory, so -# startup removes a directory only when it holds nothing but that file and that -# exact process is gone. Directories without a marker, with a dump, or owned by a -# running process are left alone, and nothing is removed when ps cannot tell. -process_identity() { - # Collapse ps's padding so the value reads back unchanged from pid. - ps -o lstart= -p "$1" 2>/dev/null | awk '{$1 = $1; print}' -} -remove_unused_heap_dump_dir() { - local dir="$1" owner_pid="" owner_start="" entries - [[ -f "${dir}/pid" ]] || return 0 - entries=$(ls -A "${dir}") - [[ ${entries} == "pid" ]] || return 0 - { read -r owner_pid; read -r owner_start; } < "${dir}/pid" - [[ -n ${owner_pid} && -n ${owner_start} ]] || return 0 - [[ $(process_identity "${owner_pid}") == "${owner_start}" ]] && return 0 - rm -f "${dir}/pid" && rmdir "${dir}" 2>/dev/null - return 0 -} -if command -v ps >/dev/null 2>&1 && [[ -n $(process_identity $$) ]]; then - for HEAP_DUMP_DIR in "${LOGS}"/heapdump_"${LAUNCH_HOST}"_*/; do - [[ -d ${HEAP_DUMP_DIR} ]] && remove_unused_heap_dump_dir "${HEAP_DUMP_DIR%/}" - done -fi +# Dump directories are never removed automatically. One stays empty until a JVM +# using it runs out of memory, and that JVM may be a computer job that outlives +# the Server, so the launcher cannot tell when a directory is safe to delete. LAUNCH_STAMP="${LAUNCH_HOST}_$(date +%Y%m%d-%H%M%S)" LAUNCH_ID="${LAUNCH_STAMP}" LAUNCH_SUFFIX=0 @@ -204,8 +180,6 @@ mkdir -p "${HEAP_DUMP_DIR}" || { report_error "Failed to create heap dump directory ${HEAP_DUMP_DIR}" exit 1 } -# The script execs java below, so $$ and its start time are the JVM's. -printf '%s\n%s\n' "$$" "$(process_identity $$)" > "${HEAP_DUMP_DIR}/pid" 2>/dev/null || true CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${HEAP_DUMP_DIR}\"" CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_ID}.log\"" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 7882d6067d..4657fbe130 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -229,13 +229,12 @@ cleanup() { if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then rm -rf "${CRASH_NAME_FIXTURES[@]}" fi - # Each launcher run creates a heap dump directory. Remove only those this - # test created and that hold nothing but their owner marker. + # Each launcher run creates a heap dump directory. Remove only the empty + # ones this test created. local dump_dir for dump_dir in "${SERVER_ROOT}"/logs/heapdump_*/; do if [[ -d "$dump_dir" ]] && ! grep -Fxq -- "$dump_dir" <<< "$PRE_EXISTING_DUMP_DIRS"; then - rm -f "${dump_dir}pid" rmdir "$dump_dir" 2>/dev/null || true fi done @@ -641,8 +640,6 @@ assert_effective_flag "$ENABLED_CAPTURE" ErrorFile "$ERROR_FILE_PATTERN" ENABLED_DUMP_DIR=$(effective_flag "$ENABLED_CAPTURE" HeapDumpPath) [[ -d "$ENABLED_DUMP_DIR" ]] || fail "launcher did not create the heap dump directory ${ENABLED_DUMP_DIR}" -[[ -f "${ENABLED_DUMP_DIR}/pid" ]] || - fail "launcher did not record the owner of ${ENABLED_DUMP_DIR}" # Child JVMs the Server starts inherit JAVA_TOOL_OPTIONS. Two JVMs started with # the launcher's options, like the Server and a computer job, must each write @@ -711,10 +708,9 @@ assert_effective_flag "$ARGFILE_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" "@${A # A launch never reuses a name already taken in logs/, even within the same # second: with a fixed clock and host name, the counter moves past an existing -# heap dump directory and crash log. Of this host's earlier dump directories, -# only one whose recorded owner process is gone and that holds no dump is -# removed; one owned by a running process, one without an owner marker, -# another host's, and one holding a dump all stay. +# heap dump directory and crash log. The launcher never removes dump +# directories: an empty one from an earlier launch may still be the target of a +# computer-job JVM that outlived its Server. MOCK_DATE_BIN="${TEMP_DIR}/mock-date-bin" mkdir -p "$MOCK_DATE_BIN" printf '#!/bin/bash\necho 20200101-000000\n' > "${MOCK_DATE_BIN}/date" @@ -731,30 +727,15 @@ add_crash_fixture() { file) : > "$path" ;; empty-dir) mkdir "$path" ;; dump-dir) mkdir "$path" && : > "${path}/java_pid1.hprof" ;; - dead-owner) - # A finished process: its PID no longer has this start time. - sleep 0 & - local dead_pid=$! - wait "$dead_pid" || true - mkdir "$path" && - printf '%s\n%s\n' "$dead_pid" "Thu Jan 1 00:00:00 1970" > "${path}/pid" ;; - live-owner) - mkdir "$path" && - printf '%s\n%s\n' "$$" "$(ps -o lstart= -p $$ | awk '{$1 = $1; print}')" \ - > "${path}/pid" ;; esac fi } USED_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000" -DEAD_OWNER_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000000" -LIVE_OWNER_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000001" -UNOWNED_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000002" +OLD_EMPTY_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000000" OTHER_HOST_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-c_20191231-000000" add_crash_fixture "$USED_DUMP_DIR" dump-dir add_crash_fixture "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log" file -add_crash_fixture "$DEAD_OWNER_DUMP_DIR" dead-owner -add_crash_fixture "$LIVE_OWNER_DUMP_DIR" live-owner -add_crash_fixture "$UNOWNED_DUMP_DIR" empty-dir +add_crash_fixture "$OLD_EMPTY_DUMP_DIR" empty-dir add_crash_fixture "$OTHER_HOST_DUMP_DIR" empty-dir UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ @@ -765,12 +746,8 @@ assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ "^${LOGS_PATTERN}/heapdump_test-pod-a_20200101-000000-2$" assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ "^${LOGS_PATTERN}/hs_err_pid%p_test-pod-a_20200101-000000-2\.log$" -[[ ! -e "$DEAD_OWNER_DUMP_DIR" ]] || - fail "launcher kept an unused heap dump directory whose owner is gone" -[[ -d "$LIVE_OWNER_DUMP_DIR" ]] || - fail "launcher removed the heap dump directory of a running process" -[[ -d "$UNOWNED_DUMP_DIR" ]] || - fail "launcher removed a heap dump directory without an owner marker" +[[ -d "$OLD_EMPTY_DUMP_DIR" ]] || + fail "launcher removed an empty heap dump directory from an earlier launch" [[ -d "$OTHER_HOST_DUMP_DIR" ]] || fail "launcher removed another host's heap dump directory" [[ -f "${USED_DUMP_DIR}/java_pid1.hprof" ]] || From 6b5171083bc34b764d9d3d4c2b625ff5fed24ba0 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Wed, 7 Oct 2026 16:07:04 +0530 Subject: [PATCH 15/23] fix(server): harden crash-file defaults after a full review pass Launcher (hugegraph-server.sh): - Create the heap dump directory after every preflight check, just before exec, so a launch that fails before Java starts leaves nothing. - Claim the name with a plain mkdir, so concurrent launches with the same host name and second get distinct names; treat a dangling symlink as taken. - When the directory cannot be created (a full volume, for example), warn and dump into logs/ instead of refusing to start. - Cap the host part at 64 characters and sanitize it with LC_ALL=C. - Skip the defaults with a warning when the logs path contains a double quote or %, which JAVA_TOOL_OPTIONS cannot carry. - Pass -Dhugegraph.bootstrap.error.log in every mode. With STDOUT_MODE the HStore image had stopped copying fatal bootstrap errors into hugegraph-server.log. - report_error writes the log file before stderr and uses printf. - The name check tolerates failglob. - Correct the comments: HotSpot truncates an existing crash log on JDK 17+, HeapDumpPath expands %p from JDK 25, and @argfiles only work in JDK_JAVA_OPTIONS or on the command line. CI: list heap dumps instead of printing them in ci-service-utils.sh, leave them out of the HStore diagnostics artifact, and keep them out of the riscv64 log grep. Tests: check JDK_JAVA_OPTIONS and _JAVA_OPTIONS as the launcher passes them, -j overriding a default, a dangling symlink, that a failed preflight leaves no directory, the mkdir-failure fallback and an unwritable logs/. Clean up only this run's directories, using a test-only host name, and remove OOM dumps on any exit. Keep the OOM runs' output for diagnosis, read flag values that contain spaces, and ignore the runner's JVM option variables. Docs: docker/README.md gains a version scope, the child-JVM-wide opt-out, a warning not to remove a running Server's directory, one log volume per pod, hostNetwork, ephemeral-storage and medium: Memory notes, and the Helm chart's missing logs volume; the Helm README points to it. --- .github/workflows/pd-store-ci.yml | 5 +- docker/README.md | 14 +- helm/hugegraph/README.md | 6 + .../assembly/static/bin/hugegraph-server.sh | 140 +++++++++------ .../src/assembly/travis/ci-service-utils.sh | 5 +- .../travis/run-native-runtime-smoke-test.sh | 9 +- .../travis/test-java-security-properties.sh | 163 ++++++++++++++---- 7 files changed, 243 insertions(+), 99 deletions(-) diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index 1a001a03bb..2c3a3b7409 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -764,8 +764,9 @@ jobs: log_dir="${service#*:}" if [[ -d "$log_dir" ]]; then mkdir -p "$DIAGNOSTICS_DIR/services/$service_name" - cp -a "$log_dir/." \ - "$DIAGNOSTICS_DIR/services/$service_name/" || true + # Skip heap dumps (logs/heapdump_*/): they can be gigabytes. + tar -C "$log_dir" --exclude='*.hprof*' -cf - . | \ + tar -C "$DIAGNOSTICS_DIR/services/$service_name" -xf - || true fi done diff --git a/docker/README.md b/docker/README.md index 6a94002166..f7a760ef9d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -232,9 +232,19 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t ### Server logs and crash files -Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. +This section describes images built from current master. Images from 1.7.0 and earlier do not set `STDOUT_MODE`: `docker logs` shows only the entrypoint's output, and the service logs stay in `logs/` inside the container. -Files stay under `/hugegraph-server/logs`: the full `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`). Each launch gets its own `heapdump_*` directory, so the Server and any JVM it starts, such as a computer job, write separate dumps. The host name (the pod name on Kubernetes) keeps pods that share one log volume from picking the same name, and a `-1`, `-2`, ... counter is added when a name is already taken. `docker restart` keeps the container filesystem, so the files survive it. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs` to keep them. A heap dump can be as large as the JVM heap, and every launch writes to a new file, so a Server that keeps running out of memory adds one heap-sized dump per restart until the volume is full. The launcher never removes old dumps or `heapdump_*` directories, whether the volume is an `emptyDir` (which also survives container restarts) or a PVC. Each start leaves a directory behind, empty unless something ran out of memory, because a computer-job JVM can keep using it after its Server exits; old ones can be deleted once no HugeGraph JVM from that launch is running. Size the volume for the number of dumps you want to keep plus the logs, move or delete old `heapdump_*` directories after each incident, or turn heap dumps off with `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which keeps the image's default `JAVA_OPTS`. Adding the flag to `JAVA_OPTS` also works, but setting `JAVA_OPTS` replaces that default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. +Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. Every start also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below. + +`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. The host name (the pod name on Kubernetes, unless the pod uses `hostNetwork`) and a `-1`, `-2`, ... counter keep names from clashing. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself. + +`docker restart` keeps the container filesystem, so the files survive it; `docker compose down` or recreating the container removes them unless `/hugegraph-server/logs` is a volume. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a shared PVC with `subPathExpr: $(POD_NAME)`: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. The Helm chart in [`helm/`](../helm/hugegraph) does not mount a logs volume yet, so with it these files are lost when a container restarts. + +A heap dump can be as large as the JVM heap. Every start uses a new directory and the JVMs a Server starts can dump too, so a Server that keeps running out of memory fills the volume with heap-sized files. Size the volume for the dumps you want to keep plus the logs, and keep it within any `ephemeral-storage` limit or `emptyDir` `sizeLimit`: eviction deletes an `emptyDir` together with the dump in it. Do not use an `emptyDir` with `medium: Memory`, which counts against the memory limit. A liveness probe that restarts the pod while a large dump is being written leaves a truncated file. + +The launcher never deletes dumps or `heapdump_*` directories. Each start leaves a directory behind, empty unless something ran out of memory; the launcher keeps it because a computer-job JVM can go on using it after its Server exits. Move or delete old directories once no HugeGraph JVM from that launch is running, but never the newest one of a running Server: without its directory, the next dump is written as a file with that name and later dumps from that launch fail. + +To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory. ## Developers diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 25ba786c34..e0d906b0ad 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -1065,6 +1065,12 @@ kubectl get pods -o wide kubectl -n describe pod | grep -A5 "Last State" ``` +`kubectl -n logs --previous` shows the previous container's +output, including launcher and bootstrap errors that stopped the Server. The +chart does not mount a volume at `/hugegraph-server/logs`, so JVM crash logs and +heap dumps the Server writes there are lost when its container restarts; see +[Server logs and crash files](../../docker/README.md#server-logs-and-crash-files). + ### Release Name Too Long Helm itself rejects release names longer than 53 characters, before this chart diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index cb84251961..b06f193b68 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -57,14 +57,15 @@ USER_OPTION="${SERVER_ARGS[3]:-}" GC_OPTION="${SERVER_ARGS[4]:-}" OPEN_TELEMETRY="${SERVER_ARGS[5]:-}" -# Fatal launcher errors go to stderr, which reaches the terminal or the container -# log, and, when possible, to ${OUTPUT}, the log start-hugegraph.sh points -# operators at. +# Launcher errors and warnings go to ${OUTPUT} (hugegraph-server.log) when it can +# be written, and to stderr, which reaches the terminal or the container log. The +# file comes first so the message is kept even if writing to stderr kills the +# script (a closed pipe, for example). report_error() { - echo "$1" >&2 - if [[ -w "${LOGS}" ]]; then - { echo "$1" >> "${OUTPUT}"; } 2>/dev/null || true + if [[ -w "${OUTPUT}" || ( ! -e "${OUTPUT}" && -w "${LOGS}" ) ]]; then + (printf '%s\n' "$1" >> "${OUTPUT}") 2>/dev/null || true fi + printf '%s\n' "$1" >&2 } ensure_path_writable "$LOGS" report_error @@ -143,48 +144,6 @@ if [ "$JAVA_OPTIONS" = "" ]; then # -Xloggc:./logs/gc.log -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps" fi -# Keep heap dumps and JVM crash logs in $LOGS whatever JAVA_OPTIONS holds. The -# defaults go first in JAVA_TOOL_OPTIONS, which the JVM reads before the operator's -# own JAVA_TOOL_OPTIONS, JDK_JAVA_OPTIONS, the command line (JAVA_OPTIONS and -j) -# and _JAVA_OPTIONS, so any of those overrides them. The JVM does that parsing -# itself, including quoted options and @argfiles. -# Child JVMs the Server starts (computer jobs, for example) inherit these options, -# so every path must stay unique per JVM. ErrorFile expands %p to each JVM's PID. -# HeapDumpPath does not expand %p, but when it names an existing directory each JVM -# writes java_pid.hprof inside it, so it points at a directory per launch. -# A restarted container often reuses the PID, and HotSpot will not overwrite an -# existing crash log or heap dump, so the names carry the host name (the pod name -# on Kubernetes, so pods sharing one log volume do not collide), the launch time, -# and a counter that skips names already used in $LOGS. -crash_files_exist() { - [[ -e "${LOGS}/heapdump_$1" ]] && return 0 - local file - for file in "${LOGS}"/hs_err_pid*_"$1".log; do - [[ -e ${file} ]] && return 0 - done - return 1 -} -LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | tr -c 'A-Za-z0-9._-' '_') -# Dump directories are never removed automatically. One stays empty until a JVM -# using it runs out of memory, and that JVM may be a computer job that outlives -# the Server, so the launcher cannot tell when a directory is safe to delete. -LAUNCH_STAMP="${LAUNCH_HOST}_$(date +%Y%m%d-%H%M%S)" -LAUNCH_ID="${LAUNCH_STAMP}" -LAUNCH_SUFFIX=0 -while crash_files_exist "${LAUNCH_ID}"; do - LAUNCH_SUFFIX=$((LAUNCH_SUFFIX + 1)) - LAUNCH_ID="${LAUNCH_STAMP}-${LAUNCH_SUFFIX}" -done -HEAP_DUMP_DIR="${LOGS}/heapdump_${LAUNCH_ID}" -mkdir -p "${HEAP_DUMP_DIR}" || { - report_error "Failed to create heap dump directory ${HEAP_DUMP_DIR}" - exit 1 -} -CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" -CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${HEAP_DUMP_DIR}\"" -CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_ID}.log\"" -export JAVA_TOOL_OPTIONS="${CRASH_OPTIONS}${JAVA_TOOL_OPTIONS:+ ${JAVA_TOOL_OPTIONS}}" - # Using G1GC as the default garbage collector (Recommended for large memory machines) # mention: zgc is only available on ARM-Mac with java > 13 case "$GC_OPTION" in @@ -284,7 +243,8 @@ if [ "${OPEN_TELEMETRY}" == "true" ]; then fi # Note: check carefully if multi "javeagent" params are set - # Append, so the crash-file defaults and the operator's JAVA_TOOL_OPTIONS stay. + # Append, so the operator's JAVA_TOOL_OPTIONS stays; the crash-file defaults + # are put in front of all of it below. export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:+${JAVA_TOOL_OPTIONS} }-javaagent:${PLUGINS}/${OT_JAR}" export OTEL_TRACES_EXPORTER=otlp export OTEL_METRICS_EXPORTER=none @@ -297,13 +257,81 @@ if [ "${OPEN_TELEMETRY}" == "true" ]; then export OTEL_RESOURCE_ATTRIBUTES=service.name=server fi -if [[ "${STDOUT_MODE:-false}" != "true" ]]; then - # Daemon stderr only reaches hugegraph-server-stdout.log, so a bootstrap - # rejection of the effective DNS policy (a broken operator override - # included) would be invisible in the log start-hugegraph.sh points - # operators at. Let the bootstrap mirror its fatal errors there. - JVM_OPTIONS="${JVM_OPTIONS} -Dhugegraph.bootstrap.error.log=${OUTPUT}" -fi +# Let the bootstrap mirror its fatal errors (a rejected DNS policy, a broken +# operator override included) into hugegraph-server.log. In daemon mode its +# stderr only reaches hugegraph-server-stdout.log; in STDOUT_MODE it reaches the +# container log, which keeps a single earlier generation, so the file copy is +# what survives a crash loop on a mounted logs volume. +JVM_OPTIONS="${JVM_OPTIONS} -Dhugegraph.bootstrap.error.log=${OUTPUT}" + +# Keep heap dumps and JVM crash logs in $LOGS whatever JAVA_OPTIONS holds. The +# defaults come first in JAVA_TOOL_OPTIONS, ahead of the operator's own value. The +# JVM applies JAVA_TOOL_OPTIONS before JDK_JAVA_OPTIONS, the command line +# (JAVA_OPTIONS and -j) and _JAVA_OPTIONS, and the last occurrence of a flag wins, +# so any of those overrides a default. The JVM does the parsing itself, including +# quoted options, and @argfiles in JDK_JAVA_OPTIONS or on the command line. +# Child JVMs the Server starts (computer jobs, for example) inherit the +# environment but not the command line, so they get these defaults too, and every +# path must stay unique per JVM. ErrorFile expands %p to each JVM's PID. +# HeapDumpPath expands %p only from JDK 25, but on every version, when it names an +# existing directory, each JVM writes java_pid.hprof inside it, so it +# points at one directory per launch. +# A restarted container often reuses the PID; HotSpot truncates an existing crash +# log (JDK 17+) and will not write a heap dump over an existing file. So the names +# carry the host name (the pod name on Kubernetes, so pods sharing one log volume +# do not collide), the launch time, and a counter. A plain mkdir claims each name +# atomically, so concurrent launches never share one. This runs after every +# preflight check, so a launch that fails before Java starts leaves nothing. +# Dump directories are never removed automatically: one stays empty until a JVM +# using it runs out of memory, and that JVM may be a computer job that outlives +# the Server, so the launcher cannot tell when a directory is safe to delete. +crash_name_taken() { + local name="$1" file restore_failglob taken=1 + [[ -e "${LOGS}/heapdump_${name}" || -L "${LOGS}/heapdump_${name}" ]] && return 0 + restore_failglob=$(shopt -p failglob) + shopt -u failglob + for file in "${LOGS}"/hs_err_pid*_"${name}".log; do + if [[ -e ${file} || -L ${file} ]]; then + taken=0 + break + fi + done + eval "${restore_failglob}" + return ${taken} +} +case "${LOGS}" in + *\"*|*%*) + # JAVA_TOOL_OPTIONS has no escape for a double quote, and ErrorFile expands %. + report_error "WARN: ${LOGS} contains a double quote or %, so the heap dump and crash log\ + defaults are not set; set -XX:HeapDumpPath and -XX:ErrorFile yourself" + ;; + *) + LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | LC_ALL=C tr -c 'A-Za-z0-9._-' '_') + LAUNCH_HOST="${LAUNCH_HOST:0:64}" + LAUNCH_STAMP="${LAUNCH_HOST}_$(date +%Y%m%d-%H%M%S)" + LAUNCH_ID="${LAUNCH_STAMP}" + LAUNCH_SUFFIX=0 + HEAP_DUMP_PATH="" + while [[ -z ${HEAP_DUMP_PATH} ]]; do + if ! crash_name_taken "${LAUNCH_ID}" && + mkdir "${LOGS}/heapdump_${LAUNCH_ID}" 2>/dev/null; then + HEAP_DUMP_PATH="${LOGS}/heapdump_${LAUNCH_ID}" + elif ! crash_name_taken "${LAUNCH_ID}" || [[ ${LAUNCH_SUFFIX} -ge 1000 ]]; then + # Not a name clash (a full disk, for example): keep starting, and dump + # into $LOGS itself rather than make diagnostics a startup requirement. + report_error "WARN: cannot create ${LOGS}/heapdump_${LAUNCH_ID}; heap dumps go to ${LOGS}" + HEAP_DUMP_PATH="${LOGS}" + else + LAUNCH_SUFFIX=$((LAUNCH_SUFFIX + 1)) + LAUNCH_ID="${LAUNCH_STAMP}-${LAUNCH_SUFFIX}" + fi + done + CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError" + CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:HeapDumpPath=${HEAP_DUMP_PATH}\"" + CRASH_OPTIONS="${CRASH_OPTIONS} \"-XX:ErrorFile=${LOGS}/hs_err_pid%p_${LAUNCH_ID}.log\"" + export JAVA_TOOL_OPTIONS="${CRASH_OPTIONS}${JAVA_TOOL_OPTIONS:+ ${JAVA_TOOL_OPTIONS}}" + ;; +esac # Turn on security check if [[ "${STDOUT_MODE:-false}" == "true" ]]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh index 934db1f171..6fc0069c0a 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh @@ -29,7 +29,10 @@ function dump_service_diagnostics() { (ss -ltnp || netstat -ltnp || true) 2>&1 if [ -d "${log_dir}" ]; then - find "${log_dir}" -maxdepth 2 -type f | sort | while read -r log_file; do + # Heap dumps under logs/heapdump_*/ are binary and can be gigabytes; + # list them instead of printing them. + find "${log_dir}" -maxdepth 2 -type f -name '*.hprof*' -exec ls -l {} + || true + find "${log_dir}" -maxdepth 2 -type f ! -name '*.hprof*' | sort | while read -r log_file; do echo "--- tail -n 200 ${log_file} ---" tail -n 200 "${log_file}" || true done diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh index e8570eef01..9814783479 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-native-runtime-smoke-test.sh @@ -78,12 +78,13 @@ start_server tee "$WORK_DIR/verify.log" grep -q '^server-e2e-smoke-verify-ok$' "$WORK_DIR/verify.log" +# Heap dumps under logs/heapdump_*/ contain class names as plain strings, so +# leave them out of the search. +LINKAGE_ERRORS='undefined symbol|UnsatisfiedLinkError|UnsupportedClassVersionError|NoClassDefFoundError' if [[ -d "$SERVER_DIR/logs" ]] && \ - grep -Eirq 'undefined symbol|UnsatisfiedLinkError|UnsupportedClassVersionError|NoClassDefFoundError' \ - "$SERVER_DIR/logs"; then + grep -Eirq --exclude='*.hprof*' "$LINKAGE_ERRORS" "$SERVER_DIR/logs"; then echo "Native linkage or Java compatibility error found in $SERVER_DIR/logs" >&2 - grep -Eirn 'undefined symbol|UnsatisfiedLinkError|UnsupportedClassVersionError|NoClassDefFoundError' \ - "$SERVER_DIR/logs" >&2 + grep -Eirn --exclude='*.hprof*' "$LINKAGE_ERRORS" "$SERVER_DIR/logs" >&2 exit 1 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 4657fbe130..8a0668df9f 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -24,6 +24,13 @@ SERVER_SCRIPT="${SERVER_ROOT}/bin/hugegraph-server.sh" CONF="${SERVER_ROOT}/conf" SECURITY_PROPERTIES="${CONF}/java-security.properties" JVM_MODULE_OPTIONS="${SERVER_ROOT}/bin/jvm-module.options" +# The test sets every JVM option source itself; values inherited from the runner +# would change what a JVM settles on. +unset JAVA_TOOL_OPTIONS JDK_JAVA_OPTIONS _JAVA_OPTIONS +# Launcher runs create heap dump directories named after the host. A host name +# unique to this run lets the cleanup tell them from a real Server's directories. +TEST_HOST="hgtest-$$" +export HOSTNAME="$TEST_HOST" fail() { echo "FAIL: $1" >&2 @@ -207,10 +214,9 @@ fi TEMP_DIR=$(mktemp -d) CRASH_NAME_FIXTURES=() +LAUNCHER_DUMP_DIRS=() OT_FIXTURE_JAR="" -# Heap dump directories that exist before the test belong to someone else (a -# running Server, say), so the cleanup below never touches them. -PRE_EXISTING_DUMP_DIRS=$(ls -d "${SERVER_ROOT}"/logs/heapdump_*/ 2>/dev/null || true) +OOM_DUMP_DIR="" SECURITY_PROPERTIES_BACKUP="${TEMP_DIR}/java-security.properties" cleanup() { @@ -229,12 +235,15 @@ cleanup() { if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then rm -rf "${CRASH_NAME_FIXTURES[@]}" fi - # Each launcher run creates a heap dump directory. Remove only the empty - # ones this test created. + if [[ -n "${OOM_DUMP_DIR:-}" ]]; then + rm -f "$OOM_DUMP_DIR"/java_pid*.hprof + fi + # Each launcher run creates a heap dump directory. Remove only the empty ones + # this run created: those named after TEST_HOST, and the few it names itself. local dump_dir - for dump_dir in "${SERVER_ROOT}"/logs/heapdump_*/; do - if [[ -d "$dump_dir" ]] && - ! grep -Fxq -- "$dump_dir" <<< "$PRE_EXISTING_DUMP_DIRS"; then + for dump_dir in "${SERVER_ROOT}"/logs/heapdump_"${TEST_HOST}"_*/ \ + ${LAUNCHER_DUMP_DIRS[@]+"${LAUNCHER_DUMP_DIRS[@]}"}; do + if [[ -d "$dump_dir" && ! -L "${dump_dir%/}" ]]; then rmdir "$dump_dir" 2>/dev/null || true fi done @@ -291,7 +300,10 @@ assert_reached_server_startup() { assert_clean_bootstrap_error() { local error_file="$1" - if grep -Eq 'Log4j|NetUtils|UnknownHost|hostname' "$error_file"; then + # Every launcher run now makes the JVM print "Picked up JAVA_TOOL_OPTIONS:", + # whose paths may contain any host or directory name. + if grep -v '^Picked up ' "$error_file" | + grep -Eq 'Log4j|NetUtils|UnknownHost|hostname'; then fail "bootstrap initialized logging or hostname resolution" fi } @@ -569,6 +581,8 @@ if [[ " $* " == *" -version "* ]]; then fi printf '%s\n' "$@" > "$CAPTURE_FILE" printf '%s\n' "${JAVA_TOOL_OPTIONS:-}" > "${CAPTURE_FILE}.tool-options" +printf '%s\n' "${JDK_JAVA_OPTIONS:-}" > "${CAPTURE_FILE}.jdk-java-options" +printf '%s\n' "${_JAVA_OPTIONS:-}" > "${CAPTURE_FILE}.underscore-java-options" MOCK chmod +x "${MOCK_JAVA_HOME}/bin/java" @@ -592,32 +606,38 @@ assert_no_argument '^-D(networkaddress\.cache\.ttl|sun\.net\.inetaddr\.ttl)=' \ # Heap dump and crash log defaults go first in JAVA_TOOL_OPTIONS, so the JVM's # own precedence lets every operator source override them. Check the values a # real JVM settles on when started the way a captured launcher run would start -# it: the captured JAVA_TOOL_OPTIONS, the crash-related -XX: arguments, and -# JDK_JAVA_OPTIONS. +# it: the JAVA_TOOL_OPTIONS, JDK_JAVA_OPTIONS and _JAVA_OPTIONS the launcher +# passed on, and the crash-related -XX: arguments of its command line. effective_flag() { local capture="$1" local flag="$2" - local jdk_java_options="${3:-}" local args=() local line + local flags while IFS= read -r line; do if [[ "$line" =~ ^-XX:([+-]HeapDumpOnOutOfMemoryError|HeapDumpPath=|ErrorFile=) ]]; then args+=("$line") fi done < "$capture" - JAVA_TOOL_OPTIONS="$(cat "${capture}.tool-options")" \ - JDK_JAVA_OPTIONS="$jdk_java_options" \ - "$JAVA_BIN" ${args[@]+"${args[@]}"} -XX:+PrintFlagsFinal -version \ - 2>/dev/null | awk -v flag="$flag" '$2 == flag { print $4; exit }' + if ! flags=$(JAVA_TOOL_OPTIONS="$(cat "${capture}.tool-options")" \ + JDK_JAVA_OPTIONS="$(cat "${capture}.jdk-java-options")" \ + _JAVA_OPTIONS="$(cat "${capture}.underscore-java-options")" \ + "$JAVA_BIN" ${args[@]+"${args[@]}"} -XX:+PrintFlagsFinal -version \ + 2>"${capture}.flags.err"); then + fail "JVM rejected the options captured in ${capture}: $(cat "${capture}.flags.err")" + fi + # The value sits between "= " and the trailing "{origin}" columns and may + # contain spaces. + printf '%s\n' "$flags" | awk -v flag="$flag" '$2 == flag { + sub(/^[^=]*= /, ""); sub(/ *(\{[^}]*\} *)+$/, ""); print; exit }' } assert_effective_flag() { local capture="$1" local flag="$2" local pattern="$3" - local jdk_java_options="${4:-}" local value - value=$(effective_flag "$capture" "$flag" "$jdk_java_options") + value=$(effective_flag "$capture" "$flag") [[ "$value" =~ $pattern ]] || fail "effective ${flag} is '${value}', expected to match ${pattern}" } @@ -625,7 +645,8 @@ assert_effective_flag() { LOGS_PATTERN=$(printf '%s' "${SERVER_ROOT}/logs" | sed 's/[][\.*^$+?(){}|]/\\&/g') # The names carry the host name, the launch time and, if needed, a counter, # since a restarted container often reuses the PID, pods may share one log -# volume, and HotSpot will not overwrite an existing crash log or heap dump. +# volume, HotSpot truncates an existing crash log (JDK 17+), and it will not +# write a heap dump over an existing file. # HeapDumpPath names a directory per launch, so each JVM writes its own # java_pid.hprof there. NAME_ID_PATTERN="[A-Za-z0-9._-]+_[0-9]{8}-[0-9]{6}(-[0-9]+)?" @@ -658,15 +679,18 @@ public class OomCheck { } } JAVA +OOM_DUMP_DIR="$ENABLED_DUMP_DIR" for OOM_RUN in 1 2; do JAVA_TOOL_OPTIONS="$(cat "${ENABLED_CAPTURE}.tool-options")" \ - "$JAVA_BIN" -Xmx64m "$OOM_SOURCE" >/dev/null 2>&1 || true + "$JAVA_BIN" -Xmx64m "$OOM_SOURCE" >"${TEMP_DIR}/oom-${OOM_RUN}.log" 2>&1 || true done -OOM_DUMPS=$(find "$ENABLED_DUMP_DIR" -maxdepth 1 -name 'java_pid*.hprof' | wc -l) -rm -f "$ENABLED_DUMP_DIR"/java_pid*.hprof +OOM_DUMPS=$(find "$OOM_DUMP_DIR" -maxdepth 1 -name 'java_pid*.hprof' | wc -l) if [[ "$OOM_DUMPS" -ne 2 ]]; then - fail "expected a heap dump per JVM in ${ENABLED_DUMP_DIR}, found ${OOM_DUMPS}" + ls -l "$OOM_DUMP_DIR" >&2 || true + cat "${TEMP_DIR}"/oom-*.log >&2 || true + fail "expected a heap dump per JVM in ${OOM_DUMP_DIR}, found ${OOM_DUMPS}" fi +rm -f "$OOM_DUMP_DIR"/java_pid*.hprof # JAVA_OPTIONS replaces the default heap options; the crash defaults stay, and # paths given there win. @@ -702,15 +726,26 @@ CAPTURE_FILE="$ARGFILE_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null -assert_effective_flag "$ARGFILE_CAPTURE" ErrorFile '^/operator/argfile-hs_err\.log$' \ - "@${ARGFILE}" -assert_effective_flag "$ARGFILE_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" "@${ARGFILE}" +assert_effective_flag "$ARGFILE_CAPTURE" ErrorFile '^/operator/argfile-hs_err\.log$' +assert_effective_flag "$ARGFILE_CAPTURE" HeapDumpPath "$HEAP_DUMP_PATTERN" + +# _JAVA_OPTIONS comes after the command line, and -j lands on the command line +# when JAVA_OPTIONS is unset; both override the defaults. +UNDERSCORE_CAPTURE="${TEMP_DIR}/underscore-java-options.args" +CAPTURE_FILE="$UNDERSCORE_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + _JAVA_OPTIONS="-XX:HeapDumpPath=/operator/underscore" \ + STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true \ + "-XX:ErrorFile=/operator/user-option-hs_err.log" >/dev/null + +assert_effective_flag "$UNDERSCORE_CAPTURE" HeapDumpPath '^/operator/underscore$' +assert_effective_flag "$UNDERSCORE_CAPTURE" ErrorFile '^/operator/user-option-hs_err\.log$' # A launch never reuses a name already taken in logs/, even within the same # second: with a fixed clock and host name, the counter moves past an existing -# heap dump directory and crash log. The launcher never removes dump -# directories: an empty one from an earlier launch may still be the target of a -# computer-job JVM that outlived its Server. +# heap dump directory, a crash log and a dangling symlink. The launcher never +# removes dump directories: an empty one from an earlier launch may still be the +# target of a computer-job JVM that outlived its Server. MOCK_DATE_BIN="${TEMP_DIR}/mock-date-bin" mkdir -p "$MOCK_DATE_BIN" printf '#!/bin/bash\necho 20200101-000000\n' > "${MOCK_DATE_BIN}/date" @@ -721,12 +756,13 @@ mkdir -p "${SERVER_ROOT}/logs" add_crash_fixture() { local path="$1" local kind="$2" - if [[ ! -e "$path" ]]; then + if [[ ! -e "$path" && ! -L "$path" ]]; then CRASH_NAME_FIXTURES+=("$path") case "$kind" in file) : > "$path" ;; empty-dir) mkdir "$path" ;; dump-dir) mkdir "$path" && : > "${path}/java_pid1.hprof" ;; + dangling-link) ln -s "${TEMP_DIR}/missing-target" "$path" ;; esac fi } @@ -735,6 +771,9 @@ OLD_EMPTY_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000000" OTHER_HOST_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-c_20191231-000000" add_crash_fixture "$USED_DUMP_DIR" dump-dir add_crash_fixture "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log" file +add_crash_fixture "${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000-2" dangling-link +LAUNCHER_DUMP_DIRS+=("${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000-3" + "${SERVER_ROOT}/logs/heapdump_test_pod-b_20200101-000000") add_crash_fixture "$OLD_EMPTY_DUMP_DIR" empty-dir add_crash_fixture "$OTHER_HOST_DUMP_DIR" empty-dir UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" @@ -743,9 +782,11 @@ CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ - "^${LOGS_PATTERN}/heapdump_test-pod-a_20200101-000000-2$" + "^${LOGS_PATTERN}/heapdump_test-pod-a_20200101-000000-3$" assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ - "^${LOGS_PATTERN}/hs_err_pid%p_test-pod-a_20200101-000000-2\.log$" + "^${LOGS_PATTERN}/hs_err_pid%p_test-pod-a_20200101-000000-3\.log$" +[[ -f "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log" ]] || + fail "launcher removed an existing crash log" [[ -d "$OLD_EMPTY_DUMP_DIR" ]] || fail "launcher removed an empty heap dump directory from an earlier launch" [[ -d "$OTHER_HOST_DUMP_DIR" ]] || @@ -768,10 +809,14 @@ fi CRASH_NAME_FIXTURES=() # A launcher error before Java starts must reach stderr, which start-hugegraph.sh -# passes to the terminal or container log, as well as the server log. +# passes to the terminal or container log, as well as the server log, and the +# failed launch must not leave a heap dump directory behind. : > "$SERVER_LOG" PREFLIGHT_ERROR="${TEMP_DIR}/preflight.err" -if JAVA_HOME="$MOCK_JAVA_HOME" STDOUT_MODE=true "$SERVER_SCRIPT" \ +PREFLIGHT_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-preflight_20200101-000000" +LAUNCHER_DUMP_DIRS+=("$PREFLIGHT_DUMP_DIR") +if JAVA_HOME="$MOCK_JAVA_HOME" HOSTNAME=test-preflight PATH="${MOCK_DATE_BIN}:${PATH}" \ + STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true "" "bad-gc" \ >/dev/null 2>"$PREFLIGHT_ERROR"; then fail "launcher accepted an unknown GC option" @@ -780,6 +825,56 @@ grep -Fq "Unrecognized gc option: 'bad-gc'" "$PREFLIGHT_ERROR" || fail "launcher preflight error did not reach stderr" grep -Fq "Unrecognized gc option: 'bad-gc'" "$SERVER_LOG" || fail "launcher preflight error did not reach the server log" +[[ ! -e "$PREFLIGHT_DUMP_DIR" ]] || + fail "a launch that failed before Java started left ${PREFLIGHT_DUMP_DIR}" + +# A full or read-only logs volume must not stop the Server: when the dump +# directory cannot be created, the launcher warns and dumps into logs/ itself. +FAILING_MKDIR_BIN="${TEMP_DIR}/failing-mkdir-bin" +mkdir -p "$FAILING_MKDIR_BIN" +cat > "${FAILING_MKDIR_BIN}/mkdir" <<'MKDIR' +#!/bin/bash +for arg in "$@"; do + case "$arg" in + */heapdump_*) echo "mkdir: ${arg}: No space left on device" >&2; exit 1 ;; + esac +done +exec /bin/mkdir "$@" +MKDIR +chmod +x "${FAILING_MKDIR_BIN}/mkdir" +: > "$SERVER_LOG" +NO_DUMP_DIR_CAPTURE="${TEMP_DIR}/no-dump-dir.args" +NO_DUMP_DIR_ERROR="${TEMP_DIR}/no-dump-dir.err" +CAPTURE_FILE="$NO_DUMP_DIR_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + PATH="${FAILING_MKDIR_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true \ + >/dev/null 2>"$NO_DUMP_DIR_ERROR" || + fail "launcher did not start when the heap dump directory could not be created" +grep -Fq "WARN: cannot create ${SERVER_ROOT}/logs/heapdump_" "$NO_DUMP_DIR_ERROR" || + fail "launcher did not warn on stderr that the heap dump directory was not created" +grep -Fq "WARN: cannot create ${SERVER_ROOT}/logs/heapdump_" "$SERVER_LOG" || + fail "launcher did not log that the heap dump directory was not created" +assert_effective_flag "$NO_DUMP_DIR_CAPTURE" HeapDumpPath "^${LOGS_PATTERN}$" + +# An unwritable logs/ is reported on stderr, where the container log keeps it. +# Root can write anywhere, so the check only runs for other users. +if [[ "$(id -u)" -ne 0 ]]; then + mkdir -p "${TEMP_DIR}/unwritable-logs-dist/logs" "${TEMP_DIR}/unwritable-logs-dist/plugins" + # The launcher reports the resolved path (TEMP_DIR is a symlink on macOS). + MINI_ROOT=$(cd "${TEMP_DIR}/unwritable-logs-dist" && pwd -P) + cp -R "${SERVER_ROOT}/bin" "${SERVER_ROOT}/conf" "${MINI_ROOT}/" + chmod 555 "${MINI_ROOT}/logs" + UNWRITABLE_ERROR="${TEMP_DIR}/unwritable-logs.err" + if JAVA_HOME="$MOCK_JAVA_HOME" STDOUT_MODE=true "${MINI_ROOT}/bin/hugegraph-server.sh" \ + "${MINI_ROOT}/conf/gremlin-server.yaml" "${MINI_ROOT}/conf/rest-server.properties" true \ + >/dev/null 2>"$UNWRITABLE_ERROR"; then + chmod 755 "${MINI_ROOT}/logs" + fail "launcher started with an unwritable logs directory" + fi + chmod 755 "${MINI_ROOT}/logs" + grep -Fq "No write permission on directory ${MINI_ROOT}/logs" "$UNWRITABLE_ERROR" || + fail "launcher did not report the unwritable logs directory on stderr" +fi # With telemetry on, the launcher appends its agent to JAVA_TOOL_OPTIONS and # keeps both the defaults and the operator's flags there. From 17d88225f79ae879d828d0ecee46675d0bf3941f Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Wed, 7 Oct 2026 21:38:15 +0530 Subject: [PATCH 16/23] fix(server): log the riscv64 libatomic error, say where to set JAVA_TOOL_OPTIONS - configure_riscv64_libatomic takes an optional error handler, like ensure_path_writable, and the launcher defines report_error before calling it, so a missing libatomic.so.1 also reaches hugegraph-server.log. Other callers keep the plain stderr message. - docker/README.md says JAVA_TOOL_OPTIONS has to be set in the container's environment (docker run -e, a Compose environment: entry, Kubernetes env); the shipped Compose files do not pass it through. - The test checks the riscv64 error reaches stderr and the server log, with uname and ldconfig mocked. --- docker/README.md | 2 +- .../assembly/static/bin/hugegraph-server.sh | 24 +++++++------- .../src/assembly/static/bin/util.sh | 8 ++++- .../travis/test-java-security-properties.sh | 33 +++++++++++++++++++ 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/docker/README.md b/docker/README.md index f7a760ef9d..2e0599caf1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -244,7 +244,7 @@ A heap dump can be as large as the JVM heap. Every start uses a new directory an The launcher never deletes dumps or `heapdump_*` directories. Each start leaves a directory behind, empty unless something ran out of memory; the launcher keeps it because a computer-job JVM can go on using it after its Server exits. Move or delete old directories once no HugeGraph JVM from that launch is running, but never the newest one of a running Server: without its directory, the next dump is written as a file with that name and later dumps from that launch fail. -To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory. +To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Set it in the container's environment, not in the host shell: `docker run -e JAVA_TOOL_OPTIONS=...`, an `environment:` entry on the Server service in Compose (the shipped Compose files do not pass it through), or `env` on the container in Kubernetes. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index b06f193b68..cadc4908f0 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -42,10 +42,21 @@ LOGS="$TOP/logs" OUTPUT=${LOGS}/hugegraph-server.log GITHUB="https://github.com" +# Launcher errors and warnings go to ${OUTPUT} (hugegraph-server.log) when it can +# be written, and to stderr, which reaches the terminal or the container log. The +# file comes first so the message is kept even if writing to stderr kills the +# script (a closed pipe, for example). +report_error() { + if [[ -w "${OUTPUT}" || ( ! -e "${OUTPUT}" && -w "${LOGS}" ) ]]; then + (printf '%s\n' "$1" >> "${OUTPUT}") 2>/dev/null || true + fi + printf '%s\n' "$1" >&2 +} + export HUGEGRAPH_HOME="$TOP" . "${BIN}"/util.sh -configure_riscv64_libatomic || exit 1 +configure_riscv64_libatomic report_error || exit 1 # Parse the server arguments in array way SERVER_ARGS=("$@") @@ -57,17 +68,6 @@ USER_OPTION="${SERVER_ARGS[3]:-}" GC_OPTION="${SERVER_ARGS[4]:-}" OPEN_TELEMETRY="${SERVER_ARGS[5]:-}" -# Launcher errors and warnings go to ${OUTPUT} (hugegraph-server.log) when it can -# be written, and to stderr, which reaches the terminal or the container log. The -# file comes first so the message is kept even if writing to stderr kills the -# script (a closed pipe, for example). -report_error() { - if [[ -w "${OUTPUT}" || ( ! -e "${OUTPUT}" && -w "${LOGS}" ) ]]; then - (printf '%s\n' "$1" >> "${OUTPUT}") 2>/dev/null || true - fi - printf '%s\n' "$1" >&2 -} - ensure_path_writable "$LOGS" report_error ensure_path_writable "$PLUGINS" report_error diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh index fb8e701bcc..738dff748f 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/util.sh @@ -25,6 +25,7 @@ function command_available() { } function configure_riscv64_libatomic() { + local error_handler="${1:-}" if [[ "$(uname -s)" != "Linux" || "$(uname -m)" != "riscv64" ]]; then return 0 fi @@ -56,7 +57,12 @@ function configure_riscv64_libatomic() { fi if [[ -z "$libatomic" ]]; then - echo "RISC-V RocksDB requires libatomic.so.1; install libatomic1" >&2 + local message="RISC-V RocksDB requires libatomic.so.1; install libatomic1" + if [[ -n "$error_handler" ]]; then + "$error_handler" "$message" + else + echo "$message" >&2 + fi return 1 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 8a0668df9f..5eb6f05801 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -828,6 +828,39 @@ grep -Fq "Unrecognized gc option: 'bad-gc'" "$SERVER_LOG" || [[ ! -e "$PREFLIGHT_DUMP_DIR" ]] || fail "a launch that failed before Java started left ${PREFLIGHT_DUMP_DIR}" +# The riscv64 libatomic check runs before anything else, and its error must also +# reach the server log. Mock uname and ldconfig so no libatomic is found; skip if +# this machine has one at a fixed path the launcher also probes. +RISCV_LIBATOMIC_FOUND="false" +for RISCV_CANDIDATE in /lib/riscv64-linux-gnu/libatomic.so.1 \ + /usr/lib/riscv64-linux-gnu/libatomic.so.1 \ + /lib64/lp64d/libatomic.so.1 /usr/lib64/lp64d/libatomic.so.1 \ + /lib64/libatomic.so.1 /usr/lib64/libatomic.so.1; do + if [[ -r "$RISCV_CANDIDATE" ]]; then + RISCV_LIBATOMIC_FOUND="true" + fi +done +if [[ "$RISCV_LIBATOMIC_FOUND" == "false" ]]; then + MOCK_RISCV_BIN="${TEMP_DIR}/mock-riscv-bin" + mkdir -p "$MOCK_RISCV_BIN" + printf '#!/bin/bash\ncase "$1" in -s) echo Linux ;; -m) echo riscv64 ;; *) echo Linux ;; esac\n' \ + > "${MOCK_RISCV_BIN}/uname" + printf '#!/bin/bash\nexit 0\n' > "${MOCK_RISCV_BIN}/ldconfig" + chmod +x "${MOCK_RISCV_BIN}/uname" "${MOCK_RISCV_BIN}/ldconfig" + : > "$SERVER_LOG" + RISCV_ERROR="${TEMP_DIR}/riscv.err" + if JAVA_HOME="$MOCK_JAVA_HOME" PATH="${MOCK_RISCV_BIN}:${PATH}" LD_PRELOAD="" \ + STDOUT_MODE=true "$SERVER_SCRIPT" \ + "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true \ + >/dev/null 2>"$RISCV_ERROR"; then + fail "launcher started on riscv64 without libatomic" + fi + grep -Fq "RISC-V RocksDB requires libatomic.so.1" "$RISCV_ERROR" || + fail "the riscv64 libatomic error did not reach stderr" + grep -Fq "RISC-V RocksDB requires libatomic.so.1" "$SERVER_LOG" || + fail "the riscv64 libatomic error did not reach the server log" +fi + # A full or read-only logs volume must not stop the Server: when the dump # directory cannot be created, the launcher warns and dumps into logs/ itself. FAILING_MKDIR_BIN="${TEMP_DIR}/failing-mkdir-bin" From 87e45f74ae6c42645ab58711d896173c4ee3da5e Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Wed, 7 Oct 2026 22:09:22 +0530 Subject: [PATCH 17/23] fix(server): address the regression check on the hardening batch - Say "all taken" when the 1000 counter values for a stamp are used, instead of naming the 1000th directory as uncreatable. - In the test, feed PrintFlagsFinal output to awk through a here-string and filter bootstrap stderr with one awk call; with a pipe, an early awk exit could kill the writer with SIGPIPE under pipefail on large output. - docker/README.md: both images declare VOLUME /hugegraph-server, so a Compose recreate keeps the files and docker compose down leaves an unattached anonymous volume until down -v; recommend a named volume or bind mount. Note that the full-volume fallback writes plain java_pid.hprof files into logs/, where a reused PID cannot write over an earlier dump. --- docker/README.md | 4 ++-- .../src/assembly/static/bin/hugegraph-server.sh | 5 ++++- .../assembly/travis/test-java-security-properties.sh | 10 ++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docker/README.md b/docker/README.md index 2e0599caf1..5e27bc2fd5 100644 --- a/docker/README.md +++ b/docker/README.md @@ -236,9 +236,9 @@ This section describes images built from current master. Images from 1.7.0 and e Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. Every start also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below. -`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. The host name (the pod name on Kubernetes, unless the pod uses `hostNetwork`) and a `-1`, `-2`, ... counter keep names from clashing. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself. +`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. The host name (the pod name on Kubernetes, unless the pod uses `hostNetwork`) and a `-1`, `-2`, ... counter keep names from clashing. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`; in that case a restarted container that reuses a PID cannot write over an earlier dump with the same name. -`docker restart` keeps the container filesystem, so the files survive it; `docker compose down` or recreating the container removes them unless `/hugegraph-server/logs` is a volume. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a shared PVC with `subPathExpr: $(POD_NAME)`: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. The Helm chart in [`helm/`](../helm/hugegraph) does not mount a logs volume yet, so with it these files are lost when a container restarts. +Both images declare `VOLUME /hugegraph-server`, so `docker restart` and a Compose recreate keep the files in that anonymous volume. `docker compose down` leaves the volume behind but the next `up` does not attach it, so the files become hard to reach and keep using disk until `docker compose down -v` deletes them; mount a named volume or a bind mount at `/hugegraph-server/logs` to keep them reachable. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a shared PVC with `subPathExpr: $(POD_NAME)`: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. The Helm chart in [`helm/`](../helm/hugegraph) does not mount a logs volume yet, so with it these files are lost when a container restarts. A heap dump can be as large as the JVM heap. Every start uses a new directory and the JVMs a Server starts can dump too, so a Server that keeps running out of memory fills the volume with heap-sized files. Size the volume for the dumps you want to keep plus the logs, and keep it within any `ephemeral-storage` limit or `emptyDir` `sizeLimit`: eviction deletes an `emptyDir` together with the dump in it. Do not use an `emptyDir` with `medium: Memory`, which counts against the memory limit. A liveness probe that restarts the pod while a large dump is being written leaves a truncated file. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index cadc4908f0..76b4e0168a 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -316,7 +316,10 @@ case "${LOGS}" in if ! crash_name_taken "${LAUNCH_ID}" && mkdir "${LOGS}/heapdump_${LAUNCH_ID}" 2>/dev/null; then HEAP_DUMP_PATH="${LOGS}/heapdump_${LAUNCH_ID}" - elif ! crash_name_taken "${LAUNCH_ID}" || [[ ${LAUNCH_SUFFIX} -ge 1000 ]]; then + elif [[ ${LAUNCH_SUFFIX} -ge 1000 ]]; then + report_error "WARN: heap dump names for ${LAUNCH_STAMP} are all taken; heap dumps go to ${LOGS}" + HEAP_DUMP_PATH="${LOGS}" + elif ! crash_name_taken "${LAUNCH_ID}"; then # Not a name clash (a full disk, for example): keep starting, and dump # into $LOGS itself rather than make diagnostics a startup requirement. report_error "WARN: cannot create ${LOGS}/heapdump_${LAUNCH_ID}; heap dumps go to ${LOGS}" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 5eb6f05801..8cfc5c2fc2 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -302,8 +302,8 @@ assert_clean_bootstrap_error() { local error_file="$1" # Every launcher run now makes the JVM print "Picked up JAVA_TOOL_OPTIONS:", # whose paths may contain any host or directory name. - if grep -v '^Picked up ' "$error_file" | - grep -Eq 'Log4j|NetUtils|UnknownHost|hostname'; then + if awk '!/^Picked up / && /Log4j|NetUtils|UnknownHost|hostname/ { found = 1 } + END { exit !found }' "$error_file"; then fail "bootstrap initialized logging or hostname resolution" fi } @@ -628,8 +628,10 @@ effective_flag() { fi # The value sits between "= " and the trailing "{origin}" columns and may # contain spaces. - printf '%s\n' "$flags" | awk -v flag="$flag" '$2 == flag { - sub(/^[^=]*= /, ""); sub(/ *(\{[^}]*\} *)+$/, ""); print; exit }' + # A here-string, not a pipe: awk exits early, which would kill printf with + # SIGPIPE under pipefail. + awk -v flag="$flag" '$2 == flag { + sub(/^[^=]*= /, ""); sub(/ *(\{[^}]*\} *)+$/, ""); print; exit }' <<< "$flags" } assert_effective_flag() { From 41764dd89abc2e33fe19afbf6bdf8e65be254cd6 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Thu, 8 Oct 2026 08:51:49 +0530 Subject: [PATCH 18/23] docs(docker): correct the pod hostname, fallback and subPathExpr notes - On Kubernetes the host name is the pod name, or spec.hostname when set; drop the hostNetwork exception and note that the counter also separates pods sharing a host name. - Call the fallback to logs/ best effort: a full volume may have no room for the dump either. - subPathExpr: $(POD_NAME) only isolates pods when POD_NAME comes in through the Downward API; show the env and volumeMounts snippet and say the Helm chart sets neither. --- docker/README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 5e27bc2fd5..d9a46879c3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -236,9 +236,23 @@ This section describes images built from current master. Images from 1.7.0 and e Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. Every start also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below. -`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. The host name (the pod name on Kubernetes, unless the pod uses `hostNetwork`) and a `-1`, `-2`, ... counter keep names from clashing. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`; in that case a restarted container that reuses a PID cannot write over an earlier dump with the same name. +`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. + +Both images declare `VOLUME /hugegraph-server`, so `docker restart` and a Compose recreate keep the files in that anonymous volume. `docker compose down` leaves the volume behind but the next `up` does not attach it, so the files become hard to reach and keep using disk until `docker compose down -v` deletes them; mount a named volume or a bind mount at `/hugegraph-server/logs` to keep them reachable. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a per-pod directory on a shared PVC: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. For a per-pod directory, pass the pod name in through the Downward API and use it in `subPathExpr` (the Helm chart sets neither): + +```yaml +env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +volumeMounts: + - name: server-logs + mountPath: /hugegraph-server/logs + subPathExpr: $(POD_NAME) +``` -Both images declare `VOLUME /hugegraph-server`, so `docker restart` and a Compose recreate keep the files in that anonymous volume. `docker compose down` leaves the volume behind but the next `up` does not attach it, so the files become hard to reach and keep using disk until `docker compose down -v` deletes them; mount a named volume or a bind mount at `/hugegraph-server/logs` to keep them reachable. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a shared PVC with `subPathExpr: $(POD_NAME)`: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. The Helm chart in [`helm/`](../helm/hugegraph) does not mount a logs volume yet, so with it these files are lost when a container restarts. +The Helm chart in [`helm/`](../helm/hugegraph) does not mount a logs volume yet, so with it these files are lost when a container restarts. A heap dump can be as large as the JVM heap. Every start uses a new directory and the JVMs a Server starts can dump too, so a Server that keeps running out of memory fills the volume with heap-sized files. Size the volume for the dumps you want to keep plus the logs, and keep it within any `ephemeral-storage` limit or `emptyDir` `sizeLimit`: eviction deletes an `emptyDir` together with the dump in it. Do not use an `emptyDir` with `medium: Memory`, which counts against the memory limit. A liveness probe that restarts the pod while a large dump is being written leaves a truncated file. From a2d97724b4941b3b18c9c8ea7ed60a47ebf2248f Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Thu, 8 Oct 2026 22:26:28 +0530 Subject: [PATCH 19/23] test(server): run the telemetry case in an isolated copy The telemetry case created plugins/opentelemetry-javaagent.jar in the supplied distribution when `[[ ! -e ]]` said it was missing. A dangling symlink there passes that check, so `: >` followed the link and created its target, possibly outside the distribution, while cleanup removed only the link. Copy bin/ and conf/ into TEMP_DIR and put the empty agent jar there, so the test never writes into the supplied plugins/. TEMP_DIR cleanup now covers the fixture, and a failing launcher run reports its stderr. --- .../travis/test-java-security-properties.sh | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 8cfc5c2fc2..058ab157f1 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -215,7 +215,6 @@ fi TEMP_DIR=$(mktemp -d) CRASH_NAME_FIXTURES=() LAUNCHER_DUMP_DIRS=() -OT_FIXTURE_JAR="" OOM_DUMP_DIR="" SECURITY_PROPERTIES_BACKUP="${TEMP_DIR}/java-security.properties" @@ -228,10 +227,7 @@ cleanup() { mv "$SECURITY_PROPERTIES_BACKUP" "$SECURITY_PROPERTIES" fi # Fixtures the test adds to the distribution; a failed step must not leave - # them behind, or a later real telemetry start would skip its download. - if [[ -n "${OT_FIXTURE_JAR:-}" ]]; then - rm -f "$OT_FIXTURE_JAR" - fi + # them behind. if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then rm -rf "${CRASH_NAME_FIXTURES[@]}" fi @@ -912,29 +908,27 @@ if [[ "$(id -u)" -ne 0 ]]; then fi # With telemetry on, the launcher appends its agent to JAVA_TOOL_OPTIONS and -# keeps both the defaults and the operator's flags there. +# keeps both the defaults and the operator's flags there. The agent jar fixture +# lives in a copy of bin/ and conf/ under TEMP_DIR, so the test never writes into +# the supplied distribution's plugins/ (or through a symlink placed there). OT_EXPECTED_MD5=$(grep -E '^ *expected_md5=' "$SERVER_SCRIPT" | cut -d'"' -f2) MOCK_MD5_BIN="${TEMP_DIR}/mock-md5-bin" mkdir -p "$MOCK_MD5_BIN" printf '#!/bin/bash\necho "%s $1"\n' "$OT_EXPECTED_MD5" > "${MOCK_MD5_BIN}/md5sum" chmod +x "${MOCK_MD5_BIN}/md5sum" -OT_JAR_PATH="${SERVER_ROOT}/plugins/opentelemetry-javaagent.jar" -if [[ ! -e "$OT_JAR_PATH" ]]; then - mkdir -p "${SERVER_ROOT}/plugins" - OT_FIXTURE_JAR="$OT_JAR_PATH" - : > "$OT_FIXTURE_JAR" -fi +TELEMETRY_ROOT="${TEMP_DIR}/telemetry-dist" +mkdir -p "${TELEMETRY_ROOT}/logs" "${TELEMETRY_ROOT}/plugins" +cp -R "${SERVER_ROOT}/bin" "${SERVER_ROOT}/conf" "${TELEMETRY_ROOT}/" +: > "${TELEMETRY_ROOT}/plugins/opentelemetry-javaagent.jar" TELEMETRY_CAPTURE="${TEMP_DIR}/telemetry.args" CAPTURE_FILE="$TELEMETRY_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ PATH="${MOCK_MD5_BIN}:${PATH}" \ JAVA_TOOL_OPTIONS="-XX:ErrorFile=/operator/hs_err.log" \ - STDOUT_MODE=true "$SERVER_SCRIPT" \ - "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true "" "" true \ - >/dev/null -if [[ -n "${OT_FIXTURE_JAR:-}" ]]; then - rm -f "$OT_FIXTURE_JAR" - OT_FIXTURE_JAR="" -fi + STDOUT_MODE=true "${TELEMETRY_ROOT}/bin/hugegraph-server.sh" \ + "${TELEMETRY_ROOT}/conf/gremlin-server.yaml" \ + "${TELEMETRY_ROOT}/conf/rest-server.properties" true "" "" true \ + >/dev/null 2>"${TEMP_DIR}/telemetry.err" || + fail "launcher failed with telemetry on: $(cat "${TEMP_DIR}/telemetry.err")" TELEMETRY_TOOL_OPTIONS=$(cat "${TELEMETRY_CAPTURE}.tool-options") [[ "$TELEMETRY_TOOL_OPTIONS" == -XX:+HeapDumpOnOutOfMemoryError\ * ]] || From adafb74e86c7708ac27b1ade82b89f3813a8d630 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Fri, 9 Oct 2026 12:15:47 +0530 Subject: [PATCH 20/23] fix(server): qualify dump guarantees and resolve the test's dist path - HotSpot names each dump java_pid.hprof, so a JVM that reuses an earlier PID within the same launch cannot write its dump. Say so in the launcher comment and docker/README.md instead of promising one dump per JVM. - docker/README.md: only errors that hugegraph-server.sh reports reach hugegraph-server.log (when logs/ is writable); earlier entrypoint steps such as init-store.sh in the HStore image report to stderr only. The empty heapdump_* directory is created unless that failed and dumps fell back to logs/. - test-java-security-properties.sh: resolve SERVER_ROOT with pwd -P, the way the launcher resolves its own location, so a distribution passed through a symlink no longer fails the path assertions. --- docker/README.md | 6 +++--- .../src/assembly/static/bin/hugegraph-server.sh | 4 +++- .../src/assembly/travis/test-java-security-properties.sh | 4 +++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index d9a46879c3..d1742b65a9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,9 +234,9 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t This section describes images built from current master. Images from 1.7.0 and earlier do not set `STDOUT_MODE`: `docker logs` shows only the entrypoint's output, and the service logs stay in `logs/` inside the container. -Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Launcher errors that stop the Server before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr as well as `hugegraph-server.log`, so when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. Every start also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below. +Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Errors that `hugegraph-server.sh` reports before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr and, when `logs/` can be written, also to `hugegraph-server.log`. Earlier entrypoint steps, such as `init-store.sh` in the HStore image, report to stderr only. Either way, when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. Every start also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below. -`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. +`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. HotSpot names each dump after the JVM's PID, so a JVM that reuses the PID of an earlier one from the same start cannot write its dump. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. Both images declare `VOLUME /hugegraph-server`, so `docker restart` and a Compose recreate keep the files in that anonymous volume. `docker compose down` leaves the volume behind but the next `up` does not attach it, so the files become hard to reach and keep using disk until `docker compose down -v` deletes them; mount a named volume or a bind mount at `/hugegraph-server/logs` to keep them reachable. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a per-pod directory on a shared PVC: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. For a per-pod directory, pass the pod name in through the Downward API and use it in `subPathExpr` (the Helm chart sets neither): @@ -258,7 +258,7 @@ A heap dump can be as large as the JVM heap. Every start uses a new directory an The launcher never deletes dumps or `heapdump_*` directories. Each start leaves a directory behind, empty unless something ran out of memory; the launcher keeps it because a computer-job JVM can go on using it after its Server exits. Move or delete old directories once no HugeGraph JVM from that launch is running, but never the newest one of a running Server: without its directory, the next dump is written as a file with that name and later dumps from that launch fail. -To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Set it in the container's environment, not in the host shell: `docker run -e JAVA_TOOL_OPTIONS=...`, an `environment:` entry on the Server service in Compose (the shipped Compose files do not pass it through), or `env` on the container in Kubernetes. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory. +To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Set it in the container's environment, not in the host shell: `docker run -e JAVA_TOOL_OPTIONS=...`, an `environment:` entry on the Server service in Compose (the shipped Compose files do not pass it through), or `env` on the container in Kubernetes. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory, unless creating it failed and dumps fell back to `logs/`. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 76b4e0168a..12f4fe86d8 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -275,7 +275,9 @@ JVM_OPTIONS="${JVM_OPTIONS} -Dhugegraph.bootstrap.error.log=${OUTPUT}" # path must stay unique per JVM. ErrorFile expands %p to each JVM's PID. # HeapDumpPath expands %p only from JDK 25, but on every version, when it names an # existing directory, each JVM writes java_pid.hprof inside it, so it -# points at one directory per launch. +# points at one directory per launch. HotSpot picks that name, so a child JVM that +# gets a PID reused within the same launch cannot write over an earlier child's +# dump; the launcher cannot rename HotSpot's dump file. # A restarted container often reuses the PID; HotSpot truncates an existing crash # log (JDK 17+) and will not write a heap dump over an existing file. So the names # carry the host name (the pod name on Kubernetes, so pods sharing one log volume diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 058ab157f1..89260241c8 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -19,7 +19,9 @@ set -euo pipefail SERVER_ROOT_INPUT="${1:?Usage: $0 PATH_TO_SERVER_DIST [SOURCE_ROOT]}" SOURCE_ROOT_INPUT="${2:-}" -SERVER_ROOT=$(cd "$SERVER_ROOT_INPUT" && pwd) +# Resolve symlinks: the launcher resolves its own location the same way, and the +# paths it passes to the JVM are compared against this one. +SERVER_ROOT=$(cd "$SERVER_ROOT_INPUT" && pwd -P) SERVER_SCRIPT="${SERVER_ROOT}/bin/hugegraph-server.sh" CONF="${SERVER_ROOT}/conf" SECURITY_PROPERTIES="${CONF}/java-security.properties" From 66fb6eb7382e7bb429a394941a7790105735e27c Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sat, 10 Oct 2026 14:07:30 +0530 Subject: [PATCH 21/23] test(server): give the test's fixture hosts run-unique names The collision, other-host and preflight cases used fixed host names (test-pod-a, test/pod-b, test-preflight) and registered the directories the launcher would create for them for removal. An empty directory with one of those names left by an interrupted run was then deleted by the next run even though that run did not create it. Derive those host names from TEST_HOST (hgtest--a and so on), so no earlier run can own them, drop the fixed-path list, and let the EXIT cleanup match only TEST_HOST and "${TEST_HOST}-*" names. docker/README.md: the "Picked up JAVA_TOOL_OPTIONS" line appears when the JVM starts; a start that fails its preflight checks exits first. --- docker/README.md | 2 +- .../travis/test-java-security-properties.sh | 37 +++++++++---------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/docker/README.md b/docker/README.md index d1742b65a9..85cfa0da26 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,7 +234,7 @@ Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore t This section describes images built from current master. Images from 1.7.0 and earlier do not set `STDOUT_MODE`: `docker logs` shows only the entrypoint's output, and the service logs stay in `logs/` inside the container. -Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Errors that `hugegraph-server.sh` reports before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr and, when `logs/` can be written, also to `hugegraph-server.log`. Earlier entrypoint steps, such as `init-store.sh` in the HStore image, report to stderr only. Either way, when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. Every start also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below. +Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Errors that `hugegraph-server.sh` reports before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr and, when `logs/` can be written, also to `hugegraph-server.log`. Earlier entrypoint steps, such as `init-store.sh` in the HStore image, report to stderr only. Either way, when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. When the JVM starts, it also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below; a start that fails its preflight checks exits before that. `/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. HotSpot names each dump after the JVM's PID, so a JVM that reuses the PID of an earlier one from the same start cannot write its dump. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 89260241c8..b10e36b4d2 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -216,7 +216,6 @@ fi TEMP_DIR=$(mktemp -d) CRASH_NAME_FIXTURES=() -LAUNCHER_DUMP_DIRS=() OOM_DUMP_DIR="" SECURITY_PROPERTIES_BACKUP="${TEMP_DIR}/java-security.properties" @@ -236,11 +235,12 @@ cleanup() { if [[ -n "${OOM_DUMP_DIR:-}" ]]; then rm -f "$OOM_DUMP_DIR"/java_pid*.hprof fi - # Each launcher run creates a heap dump directory. Remove only the empty ones - # this run created: those named after TEST_HOST, and the few it names itself. + # Each launcher run creates a heap dump directory. Every host name this run + # uses is TEST_HOST or starts with "${TEST_HOST}-", so these globs match only + # this run's directories; remove the empty ones. local dump_dir for dump_dir in "${SERVER_ROOT}"/logs/heapdump_"${TEST_HOST}"_*/ \ - ${LAUNCHER_DUMP_DIRS[@]+"${LAUNCHER_DUMP_DIRS[@]}"}; do + "${SERVER_ROOT}"/logs/heapdump_"${TEST_HOST}"-*_*/; do if [[ -d "$dump_dir" && ! -L "${dump_dir%/}" ]]; then rmdir "$dump_dir" 2>/dev/null || true fi @@ -766,26 +766,24 @@ add_crash_fixture() { esac fi } -USED_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000" -OLD_EMPTY_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-a_20191231-000000" -OTHER_HOST_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-pod-c_20191231-000000" +USED_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_${TEST_HOST}-a_20200101-000000" +OLD_EMPTY_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_${TEST_HOST}-a_20191231-000000" +OTHER_HOST_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_${TEST_HOST}-c_20191231-000000" add_crash_fixture "$USED_DUMP_DIR" dump-dir -add_crash_fixture "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log" file -add_crash_fixture "${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000-2" dangling-link -LAUNCHER_DUMP_DIRS+=("${SERVER_ROOT}/logs/heapdump_test-pod-a_20200101-000000-3" - "${SERVER_ROOT}/logs/heapdump_test_pod-b_20200101-000000") +add_crash_fixture "${SERVER_ROOT}/logs/hs_err_pid1_${TEST_HOST}-a_20200101-000000-1.log" file +add_crash_fixture "${SERVER_ROOT}/logs/heapdump_${TEST_HOST}-a_20200101-000000-2" dangling-link add_crash_fixture "$OLD_EMPTY_DUMP_DIR" empty-dir add_crash_fixture "$OTHER_HOST_DUMP_DIR" empty-dir UNIQUE_NAME_CAPTURE="${TEMP_DIR}/unique-name.args" CAPTURE_FILE="$UNIQUE_NAME_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ - HOSTNAME=test-pod-a PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ + HOSTNAME="${TEST_HOST}-a" PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null assert_effective_flag "$UNIQUE_NAME_CAPTURE" HeapDumpPath \ - "^${LOGS_PATTERN}/heapdump_test-pod-a_20200101-000000-3$" + "^${LOGS_PATTERN}/heapdump_${TEST_HOST}-a_20200101-000000-3$" assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ - "^${LOGS_PATTERN}/hs_err_pid%p_test-pod-a_20200101-000000-3\.log$" -[[ -f "${SERVER_ROOT}/logs/hs_err_pid1_test-pod-a_20200101-000000-1.log" ]] || + "^${LOGS_PATTERN}/hs_err_pid%p_${TEST_HOST}-a_20200101-000000-3\.log$" +[[ -f "${SERVER_ROOT}/logs/hs_err_pid1_${TEST_HOST}-a_20200101-000000-1.log" ]] || fail "launcher removed an existing crash log" [[ -d "$OLD_EMPTY_DUMP_DIR" ]] || fail "launcher removed an empty heap dump directory from an earlier launch" @@ -798,11 +796,11 @@ assert_effective_flag "$UNIQUE_NAME_CAPTURE" ErrorFile \ # characters unsafe in a file name are replaced. OTHER_HOST_CAPTURE="${TEMP_DIR}/other-host.args" CAPTURE_FILE="$OTHER_HOST_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ - HOSTNAME='test/pod-b' PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ + HOSTNAME="${TEST_HOST}-b/x" PATH="${MOCK_DATE_BIN}:${PATH}" STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true >/dev/null assert_effective_flag "$OTHER_HOST_CAPTURE" HeapDumpPath \ - "^${LOGS_PATTERN}/heapdump_test_pod-b_20200101-000000$" + "^${LOGS_PATTERN}/heapdump_${TEST_HOST}-b_x_20200101-000000$" if [[ ${#CRASH_NAME_FIXTURES[@]} -gt 0 ]]; then rm -rf "${CRASH_NAME_FIXTURES[@]}" fi @@ -813,9 +811,8 @@ CRASH_NAME_FIXTURES=() # failed launch must not leave a heap dump directory behind. : > "$SERVER_LOG" PREFLIGHT_ERROR="${TEMP_DIR}/preflight.err" -PREFLIGHT_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_test-preflight_20200101-000000" -LAUNCHER_DUMP_DIRS+=("$PREFLIGHT_DUMP_DIR") -if JAVA_HOME="$MOCK_JAVA_HOME" HOSTNAME=test-preflight PATH="${MOCK_DATE_BIN}:${PATH}" \ +PREFLIGHT_DUMP_DIR="${SERVER_ROOT}/logs/heapdump_${TEST_HOST}-preflight_20200101-000000" +if JAVA_HOME="$MOCK_JAVA_HOME" HOSTNAME="${TEST_HOST}-preflight" PATH="${MOCK_DATE_BIN}:${PATH}" \ STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true "" "bad-gc" \ >/dev/null 2>"$PREFLIGHT_ERROR"; then From e9dec9c5690c0fc7e530fe983a3186c4c0a42de4 Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sat, 10 Oct 2026 14:12:45 +0530 Subject: [PATCH 22/23] fix(server): keep heap dumps when the logs path has a quote or % The defaults cannot go into JAVA_TOOL_OPTIONS when the logs path holds a double quote or %, and that branch only warned, so such installs lost the -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} flags the base always passed: a regression. In that case keep the base behaviour for the Server: put both flags at the front of JAVA_OPTIONS, so an operator value there still wins, and skip only the ErrorFile default. The warning says so, the test covers a % path and a " path in isolated copies, and docker/README.md describes the case and qualifies the per-start directory sentence. --- docker/README.md | 4 ++-- .../assembly/static/bin/hugegraph-server.sh | 10 +++++--- .../travis/test-java-security-properties.sh | 24 +++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index 85cfa0da26..d86470757b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -236,7 +236,7 @@ This section describes images built from current master. Images from 1.7.0 and e Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Errors that `hugegraph-server.sh` reports before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr and, when `logs/` can be written, also to `hugegraph-server.log`. Earlier entrypoint steps, such as `init-store.sh` in the HStore image, report to stderr only. Either way, when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. When the JVM starts, it also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below; a start that fails its preflight checks exits before that. -`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. HotSpot names each dump after the JVM's PID, so a JVM that reuses the PID of an earlier one from the same start cannot write its dump. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. +`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. HotSpot names each dump after the JVM's PID, so a JVM that reuses the PID of an earlier one from the same start cannot write its dump. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. If the logs path contains a double quote or `%`, the launcher warns and keeps the 1.7.0 behaviour for the Server alone: heap dumps go to `logs/` itself, with no per-launch directory and no crash log default. Both images declare `VOLUME /hugegraph-server`, so `docker restart` and a Compose recreate keep the files in that anonymous volume. `docker compose down` leaves the volume behind but the next `up` does not attach it, so the files become hard to reach and keep using disk until `docker compose down -v` deletes them; mount a named volume or a bind mount at `/hugegraph-server/logs` to keep them reachable. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a per-pod directory on a shared PVC: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. For a per-pod directory, pass the pod name in through the Downward API and use it in `subPathExpr` (the Helm chart sets neither): @@ -258,7 +258,7 @@ A heap dump can be as large as the JVM heap. Every start uses a new directory an The launcher never deletes dumps or `heapdump_*` directories. Each start leaves a directory behind, empty unless something ran out of memory; the launcher keeps it because a computer-job JVM can go on using it after its Server exits. Move or delete old directories once no HugeGraph JVM from that launch is running, but never the newest one of a running Server: without its directory, the next dump is written as a file with that name and later dumps from that launch fail. -To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Set it in the container's environment, not in the host shell: `docker run -e JAVA_TOOL_OPTIONS=...`, an `environment:` entry on the Server service in Compose (the shipped Compose files do not pass it through), or `env` on the container in Kubernetes. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory, unless creating it failed and dumps fell back to `logs/`. +To turn heap dumps off for every JVM, set `JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError`, which also keeps the image's default `JAVA_OPTS`. Set it in the container's environment, not in the host shell: `docker run -e JAVA_TOOL_OPTIONS=...`, an `environment:` entry on the Server service in Compose (the shipped Compose files do not pass it through), or `env` on the container in Kubernetes. Putting the flag in `JAVA_OPTS` turns dumps off for the Server JVM only, because the JVMs it starts do not see its command line, and setting `JAVA_OPTS` replaces the image default (`-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 ...`), so repeat those flags. Either way, each start still creates its empty `heapdump_*` directory, unless creating it failed or the logs path contains a double quote or `%`. ## Developers diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 12f4fe86d8..76b76ea88e 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -303,9 +303,13 @@ crash_name_taken() { } case "${LOGS}" in *\"*|*%*) - # JAVA_TOOL_OPTIONS has no escape for a double quote, and ErrorFile expands %. - report_error "WARN: ${LOGS} contains a double quote or %, so the heap dump and crash log\ - defaults are not set; set -XX:HeapDumpPath and -XX:ErrorFile yourself" + # The defaults below cannot carry this path: they quote it with double quotes + # inside JAVA_TOOL_OPTIONS, and ErrorFile expands %. Keep the base behaviour + # instead: heap dumps into $LOGS on the Server's command line, ahead of + # JAVA_OPTIONS so an operator value there still wins, and no ErrorFile default. + report_error "WARN: ${LOGS} contains a double quote or %, so heap dumps go to ${LOGS}\ + without a per-launch directory and no crash log default is set; set -XX:ErrorFile yourself" + JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${JAVA_OPTIONS}" ;; *) LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | LC_ALL=C tr -c 'A-Za-z0-9._-' '_') diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index b10e36b4d2..5f13844235 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -886,6 +886,30 @@ grep -Fq "WARN: cannot create ${SERVER_ROOT}/logs/heapdump_" "$SERVER_LOG" || fail "launcher did not log that the heap dump directory was not created" assert_effective_flag "$NO_DUMP_DIR_CAPTURE" HeapDumpPath "^${LOGS_PATTERN}$" +# A logs path with a double quote or % cannot go into JAVA_TOOL_OPTIONS. Heap +# dumps must stay on there, on the command line as before this change, with only +# the crash log default left out. Each case runs in its own copy of bin/ and conf/. +for ODD_DIR_NAME in 'pct%dir' 'dq"dir'; do + mkdir -p "${TEMP_DIR}/${ODD_DIR_NAME}/logs" "${TEMP_DIR}/${ODD_DIR_NAME}/plugins" + ODD_ROOT=$(cd "${TEMP_DIR}/${ODD_DIR_NAME}" && pwd -P) + cp -R "${SERVER_ROOT}/bin" "${SERVER_ROOT}/conf" "${ODD_ROOT}/" + ODD_CAPTURE="${TEMP_DIR}/odd-path.args" + ODD_ERROR="${TEMP_DIR}/odd-path.err" + CAPTURE_FILE="$ODD_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" STDOUT_MODE=true \ + "${ODD_ROOT}/bin/hugegraph-server.sh" \ + "${ODD_ROOT}/conf/gremlin-server.yaml" "${ODD_ROOT}/conf/rest-server.properties" true \ + >/dev/null 2>"$ODD_ERROR" || + fail "launcher failed with ${ODD_DIR_NAME} in the logs path: $(cat "$ODD_ERROR")" + assert_argument "-XX:+HeapDumpOnOutOfMemoryError" "$ODD_CAPTURE" + assert_argument "-XX:HeapDumpPath=${ODD_ROOT}/logs" "$ODD_CAPTURE" + assert_no_argument '^-XX:ErrorFile=' "$ODD_CAPTURE" + if grep -q -- '-XX:' "${ODD_CAPTURE}.tool-options"; then + fail "launcher put crash defaults in JAVA_TOOL_OPTIONS for ${ODD_DIR_NAME}" + fi + grep -Fq "WARN: ${ODD_ROOT}/logs contains a double quote or %" "$ODD_ERROR" || + fail "launcher did not warn about ${ODD_DIR_NAME} in the logs path" +done + # An unwritable logs/ is reported on stderr, where the container log keeps it. # Root can write anywhere, so the check only runs for other users. if [[ "$(id -u)" -ne 0 ]]; then From d3db8e73691bc864cdee6124e779140ecaf259cd Mon Sep 17 00:00:00 2001 From: Ayan Alam Date: Sat, 10 Oct 2026 23:22:14 +0530 Subject: [PATCH 23/23] fix(server): keep the quote/% fallback in JAVA_TOOL_OPTIONS The fallback for a logs path with a double quote or % put the heap dump flags on the command line, which the JVM reads after JAVA_TOOL_OPTIONS, so the documented JAVA_TOOL_OPTIONS=-XX:-HeapDumpOnOutOfMemoryError opt-out no longer worked for the Server there. Put the flags at the front of JAVA_TOOL_OPTIONS instead, quoting the path with the quote character it does not contain (HotSpot accepts single quotes there), so every operator source overrides them again. A path with both quote characters gets only a warning. The ErrorFile default stays off for these paths. HotSpot also names crash logs only by %p, so a child JVM that reuses an earlier child's PID within one launch truncates its crash log. The launcher cannot rename it; say so next to the heap dump caveat in the comment and docker/README.md. The test checks the resolved values on a real JVM for %, ", ' with % and both-quote paths, with and without the opt-out. --- docker/README.md | 2 +- .../assembly/static/bin/hugegraph-server.sh | 28 ++++++--- .../travis/test-java-security-properties.sh | 57 ++++++++++++------- 3 files changed, 60 insertions(+), 27 deletions(-) diff --git a/docker/README.md b/docker/README.md index 9c83b39c21..e6dc6dedc6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -238,7 +238,7 @@ This section describes images built from current master. Images from 1.7.0 and e Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the HugeGraph log to container stdout. WARN and above from Hadoop, ZooKeeper, SOFA, Netty and Commons also reaches stdout. INFO from Hadoop, Netty and Commons, the audit log and the slow-query log stay in files only; ZooKeeper and SOFA log nothing below WARN. Errors that `hugegraph-server.sh` reports before Java starts, such as an unsupported JDK or too little free memory, and fatal bootstrap errors go to stderr and, when `logs/` can be written, also to `hugegraph-server.log`. Earlier entrypoint steps, such as `init-store.sh` in the HStore image, report to stderr only. Either way, when a Server exits during startup, `docker logs` or `kubectl logs --previous` shows why. When the JVM starts, it also prints `Picked up JAVA_TOOL_OPTIONS: ...` with the crash-file defaults below; a start that fails its preflight checks exits before that. -`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. HotSpot names each dump after the JVM's PID, so a JVM that reuses the PID of an earlier one from the same start cannot write its dump. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. If the logs path contains a double quote or `%`, the launcher warns and keeps the 1.7.0 behaviour for the Server alone: heap dumps go to `logs/` itself, with no per-launch directory and no crash log default. +`/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. HotSpot names each dump and crash log after the JVM's PID, so a JVM that reuses the PID of an earlier one from the same start cannot write its dump, and its crash log replaces the earlier one's. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. If the logs path contains a double quote or `%`, the launcher warns and heap dumps go to `logs/` itself, with no per-launch directory and no crash log default; the `JAVA_TOOL_OPTIONS` opt-out below still applies. A path that contains both quote characters gets no heap dump default at all. Both images declare `VOLUME /hugegraph-server`, so `docker restart` and a Compose recreate keep the files in that anonymous volume. `docker compose down` leaves the volume behind but the next `up` does not attach it, so the files become hard to reach and keep using disk until `docker compose down -v` deletes them; mount a named volume or a bind mount at `/hugegraph-server/logs` to keep them reachable. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a per-pod directory on a shared PVC: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. For a per-pod directory, pass the pod name in through the Downward API and use it in `subPathExpr` (the Helm chart sets neither): diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index 7ec381c8b8..cdea1f8ce8 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -281,7 +281,8 @@ JVM_OPTIONS="${JVM_OPTIONS} -Dhugegraph.bootstrap.error.log=${OUTPUT}" # existing directory, each JVM writes java_pid.hprof inside it, so it # points at one directory per launch. HotSpot picks that name, so a child JVM that # gets a PID reused within the same launch cannot write over an earlier child's -# dump; the launcher cannot rename HotSpot's dump file. +# dump, and its crash log truncates the earlier child's (ErrorFile only expands +# %p); the launcher cannot rename HotSpot's files. # A restarted container often reuses the PID; HotSpot truncates an existing crash # log (JDK 17+) and will not write a heap dump over an existing file. So the names # carry the host name (the pod name on Kubernetes, so pods sharing one log volume @@ -307,13 +308,26 @@ crash_name_taken() { } case "${LOGS}" in *\"*|*%*) - # The defaults below cannot carry this path: they quote it with double quotes - # inside JAVA_TOOL_OPTIONS, and ErrorFile expands %. Keep the base behaviour - # instead: heap dumps into $LOGS on the Server's command line, ahead of - # JAVA_OPTIONS so an operator value there still wins, and no ErrorFile default. - report_error "WARN: ${LOGS} contains a double quote or %, so heap dumps go to ${LOGS}\ + # The defaults below cannot carry this path: they quote it with double quotes, + # and ErrorFile expands %. Dump into $LOGS itself instead, still from the front + # of JAVA_TOOL_OPTIONS so every operator source keeps overriding it, with the + # path in whichever quote character it does not contain, and no ErrorFile. + if [[ ${LOGS} != *"'"* ]]; then + DUMP_QUOTE="'" + elif [[ ${LOGS} != *\"* ]]; then + DUMP_QUOTE='"' + else + DUMP_QUOTE="" + fi + if [[ -n ${DUMP_QUOTE} ]]; then + report_error "WARN: ${LOGS} contains a double quote or %, so heap dumps go to ${LOGS}\ without a per-launch directory and no crash log default is set; set -XX:ErrorFile yourself" - JAVA_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${JAVA_OPTIONS}" + CRASH_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError ${DUMP_QUOTE}-XX:HeapDumpPath=${LOGS}${DUMP_QUOTE}" + export JAVA_TOOL_OPTIONS="${CRASH_OPTIONS}${JAVA_TOOL_OPTIONS:+ ${JAVA_TOOL_OPTIONS}}" + else + report_error "WARN: ${LOGS} contains both quote characters, which JAVA_TOOL_OPTIONS\ + cannot carry, so no heap dump or crash log default is set; set -XX:HeapDumpPath yourself" + fi ;; *) LAUNCH_HOST=$(printf '%s' "${HOSTNAME:-localhost}" | LC_ALL=C tr -c 'A-Za-z0-9._-' '_') diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 5f13844235..d47b7e78dd 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -886,28 +886,47 @@ grep -Fq "WARN: cannot create ${SERVER_ROOT}/logs/heapdump_" "$SERVER_LOG" || fail "launcher did not log that the heap dump directory was not created" assert_effective_flag "$NO_DUMP_DIR_CAPTURE" HeapDumpPath "^${LOGS_PATTERN}$" -# A logs path with a double quote or % cannot go into JAVA_TOOL_OPTIONS. Heap -# dumps must stay on there, on the command line as before this change, with only -# the crash log default left out. Each case runs in its own copy of bin/ and conf/. -for ODD_DIR_NAME in 'pct%dir' 'dq"dir'; do +# A logs path with a double quote or % cannot use the per-launch defaults. Heap +# dumps must stay on there, still at the front of JAVA_TOOL_OPTIONS (quoted with +# the quote character the path lacks) so the documented JAVA_TOOL_OPTIONS opt-out +# keeps winning, with only the crash log default left out. A path with both quote +# characters gets the warning only. Each case runs in its own copy of bin/ and conf/. +for ODD_DIR_NAME in 'pct%dir' 'dq"dir' "sq'pct%dir" "both'q\"dir"; do mkdir -p "${TEMP_DIR}/${ODD_DIR_NAME}/logs" "${TEMP_DIR}/${ODD_DIR_NAME}/plugins" ODD_ROOT=$(cd "${TEMP_DIR}/${ODD_DIR_NAME}" && pwd -P) cp -R "${SERVER_ROOT}/bin" "${SERVER_ROOT}/conf" "${ODD_ROOT}/" - ODD_CAPTURE="${TEMP_DIR}/odd-path.args" - ODD_ERROR="${TEMP_DIR}/odd-path.err" - CAPTURE_FILE="$ODD_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" STDOUT_MODE=true \ - "${ODD_ROOT}/bin/hugegraph-server.sh" \ - "${ODD_ROOT}/conf/gremlin-server.yaml" "${ODD_ROOT}/conf/rest-server.properties" true \ - >/dev/null 2>"$ODD_ERROR" || - fail "launcher failed with ${ODD_DIR_NAME} in the logs path: $(cat "$ODD_ERROR")" - assert_argument "-XX:+HeapDumpOnOutOfMemoryError" "$ODD_CAPTURE" - assert_argument "-XX:HeapDumpPath=${ODD_ROOT}/logs" "$ODD_CAPTURE" - assert_no_argument '^-XX:ErrorFile=' "$ODD_CAPTURE" - if grep -q -- '-XX:' "${ODD_CAPTURE}.tool-options"; then - fail "launcher put crash defaults in JAVA_TOOL_OPTIONS for ${ODD_DIR_NAME}" - fi - grep -Fq "WARN: ${ODD_ROOT}/logs contains a double quote or %" "$ODD_ERROR" || - fail "launcher did not warn about ${ODD_DIR_NAME} in the logs path" + for ODD_OPT_OUT in "" "-XX:-HeapDumpOnOutOfMemoryError"; do + ODD_CAPTURE="${TEMP_DIR}/odd-path.args" + ODD_ERROR="${TEMP_DIR}/odd-path.err" + CAPTURE_FILE="$ODD_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" STDOUT_MODE=true \ + JAVA_TOOL_OPTIONS="$ODD_OPT_OUT" "${ODD_ROOT}/bin/hugegraph-server.sh" \ + "${ODD_ROOT}/conf/gremlin-server.yaml" "${ODD_ROOT}/conf/rest-server.properties" true \ + >/dev/null 2>"$ODD_ERROR" || + fail "launcher failed with ${ODD_DIR_NAME} in the logs path: $(cat "$ODD_ERROR")" + assert_no_argument '^-XX:([+-]HeapDumpOnOutOfMemoryError|HeapDumpPath=|ErrorFile=)' \ + "$ODD_CAPTURE" + assert_effective_flag "$ODD_CAPTURE" ErrorFile '^$' + case "$ODD_DIR_NAME" in + both*) + grep -Fq "WARN: ${ODD_ROOT}/logs contains both quote characters" "$ODD_ERROR" || + fail "launcher did not warn about both quote characters in the logs path" + if grep -q -- '-XX:HeapDumpPath' "${ODD_CAPTURE}.tool-options"; then + fail "launcher set a heap dump path it cannot quote for ${ODD_DIR_NAME}" + fi + ;; + *) + grep -Fq "WARN: ${ODD_ROOT}/logs contains a double quote or %" "$ODD_ERROR" || + fail "launcher did not warn about ${ODD_DIR_NAME} in the logs path" + [[ "$(effective_flag "$ODD_CAPTURE" HeapDumpPath)" == "${ODD_ROOT}/logs" ]] || + fail "heap dumps do not go to ${ODD_ROOT}/logs for ${ODD_DIR_NAME}" + if [[ -z "$ODD_OPT_OUT" ]]; then + assert_effective_flag "$ODD_CAPTURE" HeapDumpOnOutOfMemoryError '^true$' + else + assert_effective_flag "$ODD_CAPTURE" HeapDumpOnOutOfMemoryError '^false$' + fi + ;; + esac + done done # An unwritable logs/ is reported on stderr, where the container log keeps it.