diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index bd14b5448a..a386a0ca67 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -727,19 +727,44 @@ public void destroy() { } private void initMetaManager(HugeConfig conf) { + if (conf.get(ServerOptions.META_USE_CA)) { + this.ca = new K8sDriver.CA(conf.get(ServerOptions.META_CA), + conf.get(ServerOptions.META_CLIENT_CA), + conf.get(ServerOptions.META_CLIENT_KEY)); + } + connectMetaManager(conf); + } + + /** + * Connect the MetaManager under the cluster name of rest-server.properties + * (option 'cluster'). Idempotent, and it fails when the MetaManager was + * connected earlier under another name: the meta keys are prefixed with + * the cluster, so the server would otherwise read an empty tree. Called + * from HugeGraphServer before any graph is opened, because opening an + * hstore graph connects the MetaManager with the graph's 'pd.cluster' + * (default 'hg') if nothing connected it yet. With usePD=false the server + * has no cluster of its own and the graph-level binding is the only one, + * so this is a no-op there: the check must never apply to a prefix the + * server did not bind. + */ + public static void connectMetaManager(HugeConfig conf) { + if (!conf.get(ServerOptions.USE_PD)) { + return; + } + String cluster = conf.get(ServerOptions.CLUSTER); String endpoints = conf.get(ServerOptions.PD_PEERS); - boolean useCa = conf.get(ServerOptions.META_USE_CA); String ca = null; String clientCa = null; String clientKey = null; - if (useCa) { + if (conf.get(ServerOptions.META_USE_CA)) { ca = conf.get(ServerOptions.META_CA); clientCa = conf.get(ServerOptions.META_CLIENT_CA); clientKey = conf.get(ServerOptions.META_CLIENT_KEY); - this.ca = new K8sDriver.CA(ca, clientCa, clientKey); } - this.metaManager.connect(this.cluster, MetaManager.MetaDriverType.PD, - ca, clientCa, clientKey, endpoints); + MetaManager manager = MetaManager.instance(); + manager.connect(cluster, MetaManager.MetaDriverType.PD, + ca, clientCa, clientKey, endpoints); + manager.ensureCluster(cluster); } private void initK8sManagerIfNeeded(HugeConfig conf) { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java index 6af4dde99e..8f2ff95d01 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/StandardHugeGraph.java @@ -264,10 +264,25 @@ public StandardHugeGraph(HugeConfig config) { } if (isHstore()) { - // TODO: parameterize the remaining configurations - MetaManager.instance().connect("hg", MetaManager.MetaDriverType.PD, - "ca", "ca", "ca", - config.get(CoreOptions.PD_PEERS)); + MetaManager meta = MetaManager.instance(); + String cluster = config.get(CoreOptions.PD_CLUSTER); + if (!meta.isReady()) { + // Fallback for usePD=false: with usePD=true the server has + // already connected the MetaManager under ServerOptions.CLUSTER + // (the meta keys are prefixed with the cluster name) + // TODO: parameterize the remaining configurations + meta.connect(cluster, MetaManager.MetaDriverType.PD, + "ca", "ca", "ca", config.get(CoreOptions.PD_PEERS)); + } else if (config.containsKey(CoreOptions.PD_CLUSTER.name()) && + !cluster.equals(meta.cluster())) { + // The prefix is bound once per process: the server's 'cluster' + // or the first hstore graph opened wins, a later different + // 'pd.cluster' would otherwise be dropped silently + LOG.warn("Graph '{}' sets pd.cluster='{}' but the meta cluster is " + + "already bound to '{}' (keys under HUGEGRAPH/{}/); the " + + "graph's value is ignored", this.name(), cluster, + meta.cluster(), meta.cluster()); + } } try { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java index c10de4bf43..bbd634eb73 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/config/CoreOptions.java @@ -652,6 +652,18 @@ public class CoreOptions extends OptionHolder { disallowEmpty(), "127.0.0.1:8686" ); + public static final ConfigOption PD_CLUSTER = new ConfigOption<>( + "pd.cluster", + "The cluster name prefixing the meta keys in PD " + + "('HUGEGRAPH//...') when the graph itself connects the " + + "MetaManager, i.e. the server runs with usePD=false. The prefix " + + "is bound once per process: with usePD=true the server binds its " + + "own 'cluster' option first, otherwise the first hstore graph " + + "opened wins, and a different value on a later graph is ignored " + + "with a warning.", + disallowEmpty(), + "hg" + ); public static final ConfigOption MEMORY_MODE = new ConfigOption<>( "memory.mode", "The memory mode used for query in HugeGraph.", diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java index 6637baf22c..b30b505d32 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java @@ -162,7 +162,13 @@ public synchronized void connect(String cluster, MetaDriverType type, String clientKeyFile, Object... args) { E.checkArgument(cluster != null && !cluster.isEmpty(), "The cluster can't be null or empty"); - if (this.metaDriver == null) { + if (this.metaDriver != null) { + if (!cluster.equals(this.cluster)) { + LOG.warn("MetaManager is already connected to cluster '{}', " + + "ignoring the connect request for cluster '{}'", + this.cluster, cluster); + } + } else { this.cluster = cluster; switch (type) { @@ -187,6 +193,24 @@ public synchronized void connect(String cluster, MetaDriverType type, this.initManagers(this.cluster); } + /** + * Fail fast when the MetaManager was connected earlier under another + * cluster name: every meta key is prefixed with the cluster, so a server + * that expects `expected` would otherwise silently read an empty tree. + */ + public synchronized void ensureCluster(String expected) { + E.checkState(this.metaDriver != null, + "The MetaManager is not connected yet"); + if (!expected.equals(this.cluster)) { + throw new IllegalStateException(String.format( + "The MetaManager is connected to cluster '%s', but the " + + "configured cluster is '%s'; the meta keys live under " + + "'HUGEGRAPH/%s/', set the same cluster name for the " + + "server ('cluster') and the graphs ('pd.cluster')", + this.cluster, expected, this.cluster)); + } + } + private void initManagers(String cluster) { this.authMetaManager = new AuthMetaManager(this.metaDriver, cluster); this.graphMetaManager = new GraphMetaManager(this.metaDriver, cluster); diff --git a/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java index 214a09c3eb..1165fb1a93 100644 --- a/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java +++ b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/dist/HugeGraphServer.java @@ -22,6 +22,7 @@ import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.config.ServerOptions; import org.apache.hugegraph.constant.ServiceConstant; +import org.apache.hugegraph.core.GraphManager; import org.apache.hugegraph.event.EventHub; import org.apache.hugegraph.meta.MetaManager; import org.apache.hugegraph.meta.PdMetaDriver; @@ -68,6 +69,18 @@ public HugeGraphServer(String gremlinServerConf, String restServerConf) ServiceConstant.SERVICE_NAME, ServiceConstant.AUTHORITY); + // Bind the meta cluster name ('cluster' in rest-server.properties) + // before any graph is opened: prepare() below opens every graph + // in conf/graphs, and an hstore graph would otherwise connect the + // MetaManager first under its own 'pd.cluster' (default 'hg'), + // hiding the meta written under the configured cluster + if (restServerConfig.get(ServerOptions.USE_PD)) { + GraphManager.connectMetaManager(restServerConfig); + String cluster = MetaManager.instance().cluster(); + LOG.info("Meta cluster bound to '{}' (keys under HUGEGRAPH/{}/)", + cluster, cluster); + } + // Prepare GremlinServer (registers GRAPH_CREATE listener) BEFORE // RestServer starts loading graphs from PD/meta. This ensures that // graphs loaded during RestServer initialization are captured by diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 0a9c621f90..0e010ae5f7 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -52,6 +52,7 @@ import org.apache.hugegraph.unit.core.DataTypeTest; import org.apache.hugegraph.unit.core.GraphSpaceInfoLocaleTest; import org.apache.hugegraph.unit.core.GraphManagerStoresWaitTest; +import org.apache.hugegraph.unit.core.MetaManagerClusterTest; import org.apache.hugegraph.unit.core.DirectionsTest; import org.apache.hugegraph.unit.core.ExceptionTest; import org.apache.hugegraph.unit.core.GraphManagerAdminInitTest; @@ -141,6 +142,7 @@ DataTypeTest.class, GraphSpaceInfoLocaleTest.class, GraphManagerStoresWaitTest.class, + MetaManagerClusterTest.class, DirectionsTest.class, SerialEnumTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/MetaManagerClusterTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/MetaManagerClusterTest.java new file mode 100644 index 0000000000..9c6b4e8dc8 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/MetaManagerClusterTest.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.unit.core; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.hugegraph.config.CoreOptions; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.ServerOptions; +import org.apache.hugegraph.core.GraphManager; +import org.apache.hugegraph.event.EventHub; +import org.apache.hugegraph.meta.MetaDriver; +import org.apache.hugegraph.meta.MetaManager; +import org.apache.hugegraph.testutil.Assert; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * The meta keys in PD are prefixed with the cluster name, and + * MetaManager.connect() binds that name once per process. A server that + * connects under the configured 'cluster' must not be preceded by a graph + * connecting under its own 'pd.cluster', otherwise every later lookup reads + * an empty tree (1.7.0 wrote the schema under 'hg-test', master looked + * under 'hg'). + */ +public class MetaManagerClusterTest { + + /* + * connect() also rebuilds every sub-manager of the singleton, so the + * whole instance state is snapshotted and restored, not just the driver + * and the cluster; the suite must find the singleton as it left it. + */ + private final Map snapshot = new HashMap<>(); + + @Before + public void setup() throws Exception { + for (Field f : MetaManager.class.getDeclaredFields()) { + if (Modifier.isStatic(f.getModifiers())) { + continue; + } + f.setAccessible(true); + this.snapshot.put(f, f.get(MetaManager.instance())); + } + swapField("metaDriver", null); + swapField("cluster", null); + } + + @After + public void teardown() throws Exception { + for (Map.Entry e : this.snapshot.entrySet()) { + e.getKey().set(MetaManager.instance(), e.getValue()); + } + } + + @Test + public void testFirstConnectWinsAndLaterNamesAreIgnored() throws Exception { + connectWithMockDriver("hg-test"); + Assert.assertEquals("hg-test", MetaManager.instance().cluster()); + + // the graph-level fallback ('pd.cluster', default 'hg') is a no-op + MetaManager.instance().connect("hg", MetaManager.MetaDriverType.PD, + "ca", "ca", "ca", "127.0.0.1:8686"); + Assert.assertEquals("hg-test", MetaManager.instance().cluster()); + MetaManager.instance().ensureCluster("hg-test"); + } + + @Test + public void testEnsureClusterFailsOnAnotherName() throws Exception { + connectWithMockDriver("hg"); + + Assert.assertThrows(IllegalStateException.class, () -> { + MetaManager.instance().ensureCluster("hg-test"); + }, e -> { + Assert.assertContains("connected to cluster 'hg'", e.getMessage()); + Assert.assertContains("configured cluster is 'hg-test'", + e.getMessage()); + Assert.assertContains("HUGEGRAPH/hg/", e.getMessage()); + }); + } + + @Test + public void testEnsureClusterRequiresAConnection() { + Assert.assertThrows(IllegalStateException.class, () -> { + MetaManager.instance().ensureCluster("hg-test"); + }); + } + + /** + * The server binds the configured cluster before any graph is opened; + * a graph that got there first under another name is a startup error, + * not a silently empty schema. + */ + @Test + public void testServerConnectDetectsGraphLevelCluster() throws Exception { + connectWithMockDriver(CoreOptions.PD_CLUSTER.defaultValue()); + + HugeConfig conf = serverConfig(true); + Assert.assertEquals("hg-test", conf.get(ServerOptions.CLUSTER)); + Assert.assertThrows(IllegalStateException.class, () -> { + GraphManager.connectMetaManager(conf); + }, e -> { + Assert.assertContains("configured cluster is 'hg-test'", + e.getMessage()); + }); + } + + @Test + public void testServerConnectIsIdempotentUnderTheConfiguredCluster() + throws Exception { + connectWithMockDriver("hg-test"); + HugeConfig conf = serverConfig(true); + GraphManager.connectMetaManager(conf); + GraphManager.connectMetaManager(conf); + Assert.assertEquals("hg-test", MetaManager.instance().cluster()); + } + + /** + * With usePD=false the server binds no cluster of its own: an hstore graph + * opened by HugeGremlinServer.prepare() binds the MetaManager under its + * 'pd.cluster' (default 'hg'), and nothing on the server side may check + * that binding against the server's 'cluster' default ('hg-test'). + */ + @Test + public void testUsePdFalseLeavesTheGraphLevelBindingAlone() throws Exception { + connectWithMockDriver(CoreOptions.PD_CLUSTER.defaultValue()); + + HugeConfig conf = serverConfig(false); + Assert.assertFalse(conf.get(ServerOptions.USE_PD)); + Assert.assertEquals("hg-test", conf.get(ServerOptions.CLUSTER)); + GraphManager.connectMetaManager(conf); + Assert.assertEquals("hg", MetaManager.instance().cluster()); + } + + /** + * The same on the real startup path: a GraphManager built with usePD=false + * after a graph already bound 'hg' must construct without touching the + * binding (initMetaManager() only runs from loadMetaFromPD(), i.e. with + * usePD=true). + */ + @Test + public void testGraphManagerStartupWithUsePdFalseKeepsTheGraphBinding() + throws Exception { + connectWithMockDriver(CoreOptions.PD_CLUSTER.defaultValue()); + + HugeConfig conf = serverConfig(false); + GraphManager manager = new GraphManager(conf, new EventHub("cluster-test")); + try { + Assert.assertEquals("hg", MetaManager.instance().cluster()); + } finally { + manager.close(); + } + } + + @Test + public void testTeardownRestoresTheSubManagers() throws Exception { + Field f = MetaManager.class.getDeclaredField("authMetaManager"); + f.setAccessible(true); + Object before = this.snapshot.get(f); + connectWithMockDriver("hg-test"); + Assert.assertNotSame(before, f.get(MetaManager.instance())); + teardown(); + Assert.assertSame(before, f.get(MetaManager.instance())); + } + + @Test + public void testGraphLevelDefaultStaysHg() { + Assert.assertEquals("hg", CoreOptions.PD_CLUSTER.defaultValue()); + } + + /** rest-server.properties as the server sees it, with usePD set explicitly. */ + private static HugeConfig serverConfig(boolean usePd) { + PropertiesConfiguration props = new PropertiesConfiguration(); + // HugeConfig.get() casts stored values, it does not parse strings here + props.setProperty(ServerOptions.USE_PD.name(), Boolean.valueOf(usePd)); + return new HugeConfig(props); + } + + private static void connectWithMockDriver(String cluster) throws Exception { + // connect() builds a real driver, so pre-bind a mock one and the + // cluster the same way the first successful connect() would + swapField("metaDriver", Mockito.mock(MetaDriver.class)); + swapField("cluster", cluster); + MetaManager.instance().connect(cluster, MetaManager.MetaDriverType.PD, + null, null, null, "127.0.0.1:8686"); + } + + private static void swapField(String field, Object replacement) + throws Exception { + Field f = MetaManager.class.getDeclaredField(field); + f.setAccessible(true); + f.set(MetaManager.instance(), replacement); + } +} diff --git a/hugegraph-store/docs/operations-guide.md b/hugegraph-store/docs/operations-guide.md index e94bbf1555..8835dc1e48 100644 --- a/hugegraph-store/docs/operations-guide.md +++ b/hugegraph-store/docs/operations-guide.md @@ -714,6 +714,18 @@ bin/stop-hugegraph.sh bin/start-hugegraph.sh ``` +**Meta key prefix (`usePD=true`)**: the Server keeps its schema, graph spaces +and users in PD under `HUGEGRAPH//...`, where `` is the +`cluster` option of `rest-server.properties` (default `hg-test`). Release 1.7.0 +bound that name; master builds between #3008 (2026-07-10) and #3220 bound the +literal `hg` instead, because a graph was opened before the Server's own +binding. Since #3220 the Server binds `cluster` again and logs +`Meta cluster bound to '' (keys under HUGEGRAPH//)` at +startup. If a Server comes up with an empty schema after an upgrade, compare +that line with the prefix your data lives under (1.7.0: `hg-test`; a master +snapshot from that window: `hg`) and set `cluster` in `rest-server.properties` +to the prefix that holds your data. + ### Rollback Procedure If upgrade fails: