From 2174d4079d49b4f4ec67c6290eff869dd214c209 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 13 Nov 2026 11:26:15 +0530 Subject: [PATCH 1/3] fix(server): keep schema caches current across HStore Servers With several Servers on HStore, a schema change made through one Server reached the others only for adds. Updates published no clear event, removes published one only with task.sync_deletion=false, and an event sent while a Server's PD watch was reconnecting was lost. A Server that kept an old copy then wrote it back over newer schema. Complete the existing clear event instead of adding a new mechanism: - publish after every committed add, update, remove and store clear or truncate, and once when a graph is opened - KvClient.listen() accepts a callback that runs each time a watch goes live; the Server clears its V2 schema caches when it runs - a publish that fails after the schema write no longer fails the call; the graph is kept in a pending set and published again with backoff and when the watch is live again - a generation counter and a lock on the shared cache keep a read, a listing or a write-through that started before a clear from caching its value after the clear - every event clears at once, events are not merged Not covered here: a write in flight on another Server while an index is created can still miss the index, a writer that stops for good between the schema write and its publish, two Servers changing the same element at once, serving cached schema while cut off from PD, and a watch stream that dies without an error. They are documented in docs/server-runtime.md. --- docs/server-runtime.md | 49 + helm/hugegraph/docs/operations.md | 13 +- .../apache/hugegraph/ct/env/AbstractEnv.java | 15 + .../ct/node/AbstractNodeWrapper.java | 35 +- .../MultiClusterSchemaCacheTest.java | 406 +++++++ .../MultiClusterSuiteTest.java | 1 + .../apache/hugegraph/pd/client/KvClient.java | 71 +- .../hugegraph/pd/client/KvClientTest.java | 122 ++ .../cache/CachedSchemaTransactionV2.java | 555 +++++++-- .../org/apache/hugegraph/meta/MetaDriver.java | 14 + .../apache/hugegraph/meta/MetaManager.java | 10 + .../apache/hugegraph/meta/PdMetaDriver.java | 11 + .../meta/managers/GraphMetaManager.java | 6 + .../apache/hugegraph/unit/UnitTestSuite.java | 2 + .../cache/CachedSchemaTransactionTest.java | 24 +- .../cache/CachedSchemaTransactionV2Test.java | 1002 +++++++++++++++++ 16 files changed, 2217 insertions(+), 119 deletions(-) create mode 100644 hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java create mode 100644 hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java diff --git a/docs/server-runtime.md b/docs/server-runtime.md index 3e306ec6c9..ba92d299b5 100644 --- a/docs/server-runtime.md +++ b/docs/server-runtime.md @@ -825,6 +825,55 @@ connection or delivery failures are logged. This lifecycle handling does not pro acknowledged delivery or a distributed cache consistency protocol. Validate actual DML and cache behavior against the deployment's shared backend and topology. +## Schema caches across HStore Servers + +With HStore, each Server keeps the schema of a graph in memory and never expires it. +The schema itself is stored in PD. After a Server writes a schema change to PD, it +publishes a clear event for that graph; every other Server then drops what it cached +for the graph and reads PD on the next use. + +What a deployment can rely on once every Server runs this version: + +- Every committed schema change is published after its write: creating, updating and + removing property keys, labels and index labels, the status changes made by index and + label jobs, and clearing or truncating a graph. A removal is published whatever + `task.sync_deletion` is set to. +- A Server whose PD watch was down drops all its cached schema when the watch is back, + for example after a PD restart or a PD leader change. Events published in between + are not replayed, and this is how they are made up for. +- A value read from PD before a clear is never cached after that clear. +- A clear event that could not be published does not fail the schema change, because + the change is already committed. The Server publishes it again after 1 second, then + with a delay that doubles up to 30 seconds, and at once when its PD watch is back. + Each Server also publishes once when it opens a graph, which covers a Server that + stopped between a schema write and its event. + +What it does not provide: + +- Writes during index creation. A Server learns about a new index when it handles the + event for the label that owns the index. A vertex or edge write that such a Server + prepared before that, and that reaches storage after the index build has read that + part of the label, is missing from the index. Normally this is a write in flight for + a few milliseconds. It can be longer on a Server whose PD watch is down while the + index is created. Create indexes while writes to the label are stopped, or rebuild + the index afterwards, when a complete index is required. +- Two Servers changing the same schema element at the same time. An update writes the + whole element, so one of the two changes can be lost. Send schema changes for one + element through one client, and wait for each to return before sending the next. +- A Server that stops for good between a schema write and its event. The other Servers + keep what they cached until the next schema change of that graph or their next watch + restart. +- A Server that cannot reach PD keeps serving its cached schema until it reconnects. +- A PD watch that dies without an error is noticed only when another PD call from that + Server fails. +- With the etcd meta driver the watch does not report restarts. A Server then does + not drop its caches after a watch restart, and a failed publish is sent again by the + timed retry only. + +A Server on an older version publishes nothing on update, so finish upgrading every +Server before relying on this. Index entries skipped by an older version stay missing: +rebuild indexes that were created on a cluster with several Servers. + ## Deleting a graph with local tasks Stop application writes before deletion and keep them stopped during retries. The local diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index 3996afca55..e56c265fc0 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -526,11 +526,14 @@ the full index cache. This does not detect missing/stale label associations, orphaned indexes or incorrect index data, and does not prove range-query results. Run it after index creation/rebuild jobs finish, with no schema changes during the test. Like indexed queries, the check rejects every other status, including -`CREATING`, `REBUILDING` and `INVALID`. On HStore, reading an unfinished index can -cache its temporary status on Pods other than the job runner. Later attempts -reuse that cache; the retry window does not repair it. Indexed queries on those -Pods may keep failing after the job succeeds; replace affected Server Pods only -after the jobs finish. This check does not fix cross-Server schema cache consistency. +`CREATING`, `REBUILDING` and `INVALID`. On HStore, the Pod that runs the job +publishes each status change, and the other Pods drop their cached schema and read +it again, so they see `CREATED` shortly after the job succeeds. A Pod whose PD +watch is down sees it when the watch is back. This check does not prove that, and +it does not cover the limits listed under +[Schema caches across HStore Servers](../../../docs/server-runtime.md#schema-caches-across-hstore-servers). +An index created before this behavior on a cluster with several Server Pods may +miss entries; rebuild it after the upgrade. For a graph with existing test data, provide a read-only Gremlin script that asserts the expected result. For example, with 40 existing `person` vertices diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java index ccf0582a00..8e1311a0e5 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java @@ -146,6 +146,21 @@ private static void sleepBeforeRetry() { } } + /** + * Restart one PD node on its own data and return without waiting for it. + * Restarting the PD leader makes the leader change and every PD watch + * start again, which is what a test of that window needs to race. + */ + public void restartPDNode(int index) { + PDNodeWrapper node = this.pdNodeWrappers.get(index); + System.out.printf("[cluster-test] restarting %s%n", node.getID()); + node.restart(); + } + + public void awaitPDNode(int index) { + waitUntilStarted(this.pdNodeWrappers.get(index)); + } + public void stopCluster() { for (ServerNodeWrapper serverNodeWrapper : serverNodeWrappers) { serverNodeWrapper.stop(); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java index 3f71847bbc..e6e03d1270 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -55,6 +56,8 @@ public abstract class AbstractNodeWrapper implements BaseNodeWrapper { protected int index; protected List fileNames; protected String startLine; + // Where in the startup log the running process began to write + private long startupLogOffset; public AbstractNodeWrapper() { this.clusterIndex = 1; @@ -154,8 +157,10 @@ public boolean isStarted() { return false; } - try (Scanner sc = new Scanner(Paths.get(getLogPath()), - StandardCharsets.UTF_8.name())) { + try (InputStream log = Files.newInputStream(Paths.get(getLogPath()))) { + // Skip what an earlier run of this node wrote before a restart + log.skipNBytes(this.startupLogOffset); + Scanner sc = new Scanner(log, StandardCharsets.UTF_8.name()); while (sc.hasNextLine()) { String line = sc.nextLine(); if (line.contains(startLine)) return true; @@ -165,6 +170,32 @@ public boolean isStarted() { return false; } + /** + * Stop the process and start it again in the same directory, so the node + * comes back with its data. {@link #stop()} is for the end of a test: it + * also removes the directory. + */ + public void restart() { + if (this.instance != null && this.instance.isAlive()) { + this.instance.destroy(); + try { + if (!this.instance.waitFor(20, TimeUnit.SECONDS)) { + this.instance.destroyForcibly().waitFor(10, TimeUnit.SECONDS); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted while stopping " + getID(), e); + } + } + try { + Path log = Paths.get(getLogPath()); + this.startupLogOffset = Files.isRegularFile(log) ? Files.size(log) : 0L; + } catch (IOException e) { + throw new AssertionError("Can't read the startup log of " + getID(), e); + } + this.start(); + } + protected String moduleOptions(Path options) throws IOException { if (!Files.isRegularFile(options) || !Files.isReadable(options)) { throw new IOException("Missing or unreadable JVM module options: " + options); diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java new file mode 100644 index 0000000000..1dcc478210 --- /dev/null +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.MultiClusterTest; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.SimpleClusterTest.BaseSimpleTest.RestClient; +import org.apache.hugegraph.ct.env.AbstractEnv; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.Response; + +/** + * Schema changes made through one Server must reach the schema caches of the + * other Servers (apache/hugegraph#3235). Every test changes the schema through + * one Server and then asks the other two, which have the old schema cached. + *

+ * Not covered here, because the code does not provide it: two Servers changing + * the same schema element at the same moment, and a write that is in flight on + * another Server at the instant an index is created. + */ +public class MultiClusterSchemaCacheTest extends BaseMultiClusterTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String PKS = URL_PREFIX + "/schema/propertykeys"; + private static final String VLS = URL_PREFIX + "/schema/vertexlabels"; + private static final String ILS = URL_PREFIX + "/schema/indexlabels"; + private static final String VERTICES = URL_PREFIX + "/graph/vertices"; + private static final String TASKS = URL_PREFIX + "/tasks"; + private static final long VISIBLE_SECONDS = 30L; + + @BeforeClass + public static void waitForGraphOnEveryServer() throws Exception { + waitFor("the graph to open on every Server", 120L, () -> { + for (RestClient client : clients) { + if (status(client.get(URL_PREFIX)) != 200) { + return false; + } + } + return true; + }); + } + + @Test + public void testUpdateReachesEveryServer() throws Exception { + String pk = "sc_upd_note"; + createPropertyKey(0, pk, "TEXT"); + for (RestClient client : clients) { + json(client.get(PKS + "/" + pk)); + } + + ok(put(clients.get(0), PKS + "/" + pk, "append", + "{\"name\": \"" + pk + "\", \"user_data\": {\"marker\": \"set\"}}")); + + waitFor("the appended user_data on every Server", VISIBLE_SECONDS, () -> { + for (RestClient client : clients) { + if (!json(client.get(PKS + "/" + pk)).path("user_data").has("marker")) { + return false; + } + } + return true; + }); + } + + @Test + public void testAppendThroughAnotherServerKeepsTheFirstAppend() throws Exception { + String a = "sc_app_a"; + String p1 = "sc_app_p1"; + String p2 = "sc_app_p2"; + String vl = "sc_app_person"; + for (String pk : new String[]{a, p1, p2}) { + createPropertyKey(0, pk, "TEXT"); + } + createVertexLabel(0, vl, a); + for (RestClient client : clients) { + json(client.get(VLS + "/" + vl)); + } + + ok(put(clients.get(0), VLS + "/" + vl, "append", appendBody(vl, p1))); + waitFor(p1 + " on every Server", VISIBLE_SECONDS, + () -> everyServerHasProperties(vl, a, p1)); + + // Server 1 writes the whole label back with its own change added + ok(put(clients.get(1), VLS + "/" + vl, "append", appendBody(vl, p2))); + waitFor(p1 + " and " + p2 + " on every Server", VISIBLE_SECONDS, + () -> everyServerHasProperties(vl, a, p1, p2)); + } + + @Test + public void testIndexCreatedThroughOneServerWorksOnTheOthers() throws Exception { + int indexes = 4; + String name = "sc_idx_name"; + String vl = "sc_idx_person"; + List props = new ArrayList<>(); + createPropertyKey(0, name, "TEXT"); + for (int k = 0; k < indexes; k++) { + props.add("sc_idx_c" + k); + createPropertyKey(0, props.get(k), "INT"); + } + createVertexLabel(0, vl, name, props.toArray(new String[0])); + for (int i = 0; i < 20; i++) { + StringBuilder body = new StringBuilder("{\"label\": \"" + vl + + "\", \"properties\": {\"" + name + + "\": \"v" + i + "\""); + for (String prop : props) { + body.append(", \"").append(prop).append("\": ").append(i); + } + ok(clients.get(0).post(VERTICES, body.append("}}").toString())); + } + + for (int k = 0; k < indexes; k++) { + String prop = props.get(k); + String il = "sc_idx_by_c" + k; + for (RestClient client : clients) { + json(client.get(VLS + "/" + vl)); + json(client.get(ILS)); + } + + // The other Servers serve reads of this label while the index is created + AtomicBoolean stop = new AtomicBoolean(false); + ExecutorService readers = Executors.newFixedThreadPool(4); + for (int reader = 0; reader < 4; reader++) { + RestClient client = clients.get(1 + reader % 2); + readers.submit(() -> { + while (!stop.get()) { + Response r = client.target().path(VERTICES) + .queryParam("label", vl) + .queryParam("limit", 3).request().get(); + r.readEntity(String.class); + } + }); + } + try { + JsonNode created = json(clients.get(0).post(ILS, + "{\"name\": \"" + il + "\", \"base_type\": \"VERTEX_LABEL\", " + + "\"base_value\": \"" + vl + "\", \"index_type\": \"SECONDARY\", " + + "\"fields\": [\"" + prop + "\"]}")); + String task = created.path("task_id").asText(); + waitFor("the rebuild task of " + il, 120L, () -> + "success".equals(json(clients.get(0).get(TASKS + "/" + task)) + .path("task_status").asText())); + } finally { + stop.set(true); + readers.shutdown(); + Assert.assertTrue(readers.awaitTermination(60L, TimeUnit.SECONDS)); + } + + waitFor(il + " created and listed in its label on every Server", + VISIBLE_SECONDS, () -> { + for (RestClient client : clients) { + JsonNode index = json(client.get(ILS + "/" + il)); + if (!"CREATED".equals(index.path("status").asText())) { + return false; + } + JsonNode label = json(client.get(VLS + "/" + vl)); + if (!contains(label.path("index_labels"), il)) { + return false; + } + } + return true; + }); + + // A vertex written through any Server is found through the index on any Server + for (int writer = 0; writer < clients.size(); writer++) { + ok(clients.get(writer).post(VERTICES, + "{\"label\": \"" + vl + "\", \"properties\": {\"" + name + + "\": \"w" + k + "_" + writer + "\", \"" + prop + "\": " + + value(k, writer) + "}}")); + } + int trial = k; + waitFor("vertices written through every Server in index " + il, + VISIBLE_SECONDS, () -> { + for (RestClient reader : clients) { + for (int writer = 0; writer < clients.size(); writer++) { + Response r = reader.target().path(VERTICES) + .queryParam("label", vl) + .queryParam("properties", "{condition}") + .resolveTemplate("condition", + "{\"" + prop + "\": " + + value(trial, writer) + "}") + .request().get(); + if (r.getStatus() != 200) { + r.readEntity(String.class); + return false; + } + if (json(r).path("vertices").size() != 1) { + return false; + } + } + } + return true; + }); + } + } + + @Test + public void testRemoveReachesEveryServerWithSyncDeletion() throws Exception { + /* + * task.sync_deletion used to switch the clear event of a removal off. + * This checks the removal end to end, but it is not the regression + * test for that: the Servers also publish clear events while they set + * up a new graph, which can clear the caches here on their own. The + * unit test of CachedSchemaTransactionV2 pins the publish itself. + */ + String graph = "schemacachesync"; + String prefix = "graphspaces/DEFAULT/graphs/" + graph; + String pks = prefix + "/schema/propertykeys"; + if (status(clients.get(0).get(prefix)) != 200) { + ok(clients.get(0).post(prefix, + "{\"backend\": \"hstore\", \"serializer\": \"binary\", " + + "\"store\": \"" + graph + "\", " + + "\"task.sync_deletion\": \"true\"}")); + } + waitFor("the second graph on every Server", 120L, () -> { + for (RestClient client : clients) { + if (status(client.get(prefix)) != 200) { + return false; + } + } + return true; + }); + + String pk = "sc_rm_note"; + ok(clients.get(0).post(pks, propertyKeyBody(pk, "TEXT"))); + waitFor(pk + " listed on every Server", VISIBLE_SECONDS, + () -> everyServerLists(pks, pk, true)); + + Response removed = clients.get(0).target().path(pks + "/" + pk).request().delete(); + ok(removed); + waitFor(pk + " gone from every Server", VISIBLE_SECONDS, + () -> everyServerLists(pks, pk, false)); + } + + @Test + public void testChangeDuringPDRestartReachesEveryServer() throws Exception { + AbstractEnv cluster = (AbstractEnv) env; + int pdNodes = cluster.getPDRestAddrs().size(); + // One of the PD nodes is the leader: restarting it restarts every watch + for (int node = 0; node < pdNodes; node++) { + String pk = "sc_gap_" + node; + for (RestClient client : clients) { + json(client.get(PKS)); + } + + cluster.restartPDNode(node); + // One change as early as PD takes it, while the watches may still be down + waitFor("an add right after restarting PD node " + node, 180L, () -> { + Response r = clients.get(0).post(PKS, propertyKeyBody(pk, "TEXT")); + String content = r.readEntity(String.class); + return r.getStatus() / 100 == 2 || + (r.getStatus() == 400 && content.contains("exist")); + }); + cluster.awaitPDNode(node); + + waitFor(pk + " listed on every Server after restarting PD node " + node, + 90L, () -> everyServerLists(PKS, pk, true)); + } + } + + // ------------------------------------------------------------ helpers + + private static int value(int trial, int writer) { + return 100000 + 10 * trial + writer; + } + + private static void createPropertyKey(int server, String name, String dataType) { + ok(clients.get(server).post(PKS, propertyKeyBody(name, dataType))); + } + + private static String propertyKeyBody(String name, String dataType) { + return "{\"name\": \"" + name + "\", \"data_type\": \"" + dataType + + "\", \"cardinality\": \"SINGLE\"}"; + } + + private static void createVertexLabel(int server, String name, String first, + String... others) { + StringBuilder props = new StringBuilder("\"" + first + "\""); + for (String other : others) { + props.append(", \"").append(other).append("\""); + } + ok(clients.get(server).post(VLS, + "{\"name\": \"" + name + "\", \"id_strategy\": " + + "\"AUTOMATIC\", \"properties\": [" + props + + "], \"nullable_keys\": [" + props + "]}")); + } + + private static String appendBody(String label, String property) { + return "{\"name\": \"" + label + "\", \"properties\": [\"" + property + + "\"], \"nullable_keys\": [\"" + property + "\"]}"; + } + + private static boolean everyServerHasProperties(String label, String... properties) + throws Exception { + for (RestClient client : clients) { + JsonNode found = json(client.get(VLS + "/" + label)).path("properties"); + for (String property : properties) { + if (!contains(found, property)) { + return false; + } + } + } + return true; + } + + private static boolean everyServerLists(String path, String name, boolean listed) + throws Exception { + for (RestClient client : clients) { + Response r = client.get(path); + if (r.getStatus() != 200) { + r.readEntity(String.class); + return false; + } + boolean found = false; + for (JsonNode pk : json(r).path("propertykeys")) { + found |= name.equals(pk.path("name").asText()); + } + if (found != listed) { + return false; + } + } + return true; + } + + private static boolean contains(JsonNode array, String text) { + for (JsonNode item : array) { + if (text.equals(item.asText())) { + return true; + } + } + return false; + } + + private static Response put(RestClient client, String path, String action, String body) { + return client.target().path(path).queryParam("action", action) + .request().put(Entity.json(body)); + } + + private static int status(Response response) { + response.readEntity(String.class); + return response.getStatus(); + } + + private static void ok(Response response) { + String content = response.readEntity(String.class); + Assert.assertEquals("Response with status " + response.getStatus() + + " and content " + content, 2, response.getStatus() / 100); + } + + private static JsonNode json(Response response) throws Exception { + String content = response.readEntity(String.class); + Assert.assertEquals("Response with status " + response.getStatus() + + " and content " + content, 2, response.getStatus() / 100); + return MAPPER.readTree(content); + } + + private static void waitFor(String what, long seconds, Callable condition) + throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(seconds); + Throwable last = null; + while (System.nanoTime() < deadline) { + try { + if (condition.call()) { + return; + } + last = null; + } catch (Exception | AssertionError e) { + // A Server may answer with an error while PD restarts + last = e; + } + Thread.sleep(200L); + } + try { + env.dumpClusterStatus(); + } catch (Throwable ignored) { + // The failure below is the one to report + } + throw new AssertionError("Timed out after " + seconds + " s waiting for " + what, last); + } +} diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java index 6e55cdd200..5ec39ec4e0 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSuiteTest.java @@ -26,6 +26,7 @@ @Suite.SuiteClasses({ MultiClusterDeployTest.class, MultiClusterFileTest.class, + MultiClusterSchemaCacheTest.class, }) @Slf4j public class MultiClusterSuiteTest { diff --git a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java index fb5c589053..0e63da9a51 100644 --- a/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java +++ b/hugegraph-pd/hg-pd-client/src/main/java/org/apache/hugegraph/pd/client/KvClient.java @@ -209,9 +209,19 @@ private StreamObserver getObserver(WatchSubscription subscription return new StreamObserver() { @Override public void onNext(WatchResponse value) { - if (!acceptFirstFrame(subscription, this)) { + FrameOrder order = acceptFrame(subscription, this); + if (order == FrameOrder.STALE) { return; } + if (order == FrameOrder.FIRST) { + /* + * PD registers the watcher before it sends the first frame, + * so from here on no event is missed. Anything committed + * while the watch was down was never delivered: tell the + * subscriber before any later frame is handled. + */ + notifyWatchStarted(subscription); + } switch (value.getState()) { case Starting: boolean b = subscription.clientId.compareAndSet(0, value.getClientId()); @@ -258,6 +268,24 @@ public void listen(String key, Consumer consumer, listen(key, consumer, errorConsumer, false); } + /** + * Watch a key and get told each time the watch becomes live. + * + * @param startedCallback runs on the watch thread when the watch starts and + * every time it starts again after a disconnect or a + * PD leader change. Events committed while the watch + * was down are not replayed, so a subscriber that + * keeps state derived from them must rebuild it here. + * It runs before any event of the new watch is + * delivered and must not block. + */ + public void listen(String key, Consumer consumer, + Consumer errorConsumer, + Runnable startedCallback) throws PDException { + Objects.requireNonNull(startedCallback, "startedCallback"); + listen(key, consumer, errorConsumer, startedCallback, false); + } + public void listenPrefix(String prefix, Consumer consumer) throws PDException { listen(prefix, consumer, throwable -> { }, true); } @@ -270,11 +298,19 @@ public void listenPrefix(String prefix, Consumer consumer, private void listen(String key, Consumer consumer, Consumer errorConsumer, boolean prefix) throws PDException { + listen(key, consumer, errorConsumer, null, prefix); + } + + private void listen(String key, Consumer consumer, + Consumer errorConsumer, + Runnable startedCallback, + boolean prefix) throws PDException { Objects.requireNonNull(key, "key"); Objects.requireNonNull(consumer, "consumer"); Objects.requireNonNull(errorConsumer, "errorConsumer"); WatchSubscription subscription = - new WatchSubscription(key, consumer, errorConsumer, prefix); + new WatchSubscription(key, consumer, errorConsumer, + startedCallback, prefix); subscriptions.add(subscription); try { if (!startWatch(subscription)) { @@ -358,15 +394,28 @@ private void recordAttemptChannel(WatchSubscription subscription, } } - private boolean acceptFirstFrame(WatchSubscription subscription, - StreamObserver sourceObserver) { + private FrameOrder acceptFrame(WatchSubscription subscription, + StreamObserver sourceObserver) { synchronized (subscription) { if (subscription.observer.get() != sourceObserver) { - return false; + return FrameOrder.STALE; } + boolean first = !subscription.firstFrameReceived; subscription.firstFrameReceived = true; cancelStartTimeout(subscription); - return true; + return first ? FrameOrder.FIRST : FrameOrder.LATER; + } + } + + private void notifyWatchStarted(WatchSubscription subscription) { + Runnable callback = subscription.startedCallback; + if (callback == null) { + return; + } + try { + callback.run(); + } catch (RuntimeException e) { + log.warn("The watch-started callback for key {} failed", subscription.key, e); } } @@ -728,11 +777,19 @@ public void close() { super.close(); } + /** Where a frame falls in the life of the watch that delivered it. */ + private enum FrameOrder { + STALE, + FIRST, + LATER + } + private final class WatchSubscription { private final String key; private final Consumer consumer; private final Consumer errorConsumer; + private final Runnable startedCallback; private final boolean prefix; private final AtomicReference> observer; private final AtomicBoolean reconnectScheduled; @@ -746,10 +803,12 @@ private final class WatchSubscription { private WatchSubscription(String key, Consumer consumer, Consumer errorConsumer, + Runnable startedCallback, boolean prefix) { this.key = key; this.consumer = consumer; this.errorConsumer = errorConsumer; + this.startedCallback = startedCallback; this.prefix = prefix; this.observer = new AtomicReference<>(); this.reconnectScheduled = new AtomicBoolean(false); diff --git a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java index 8c28f854e8..8724fcd6ad 100644 --- a/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java +++ b/hugegraph-pd/hg-pd-test/src/main/java/org/apache/hugegraph/pd/client/KvClientTest.java @@ -543,6 +543,128 @@ public void testFirstFramePreventsWatchTimeoutReconnect() throws Exception { } } + @Test + public void testWatchStartedCallbackRunsOnceWhenTheWatchGoesLive() throws Exception { + try (WatchTestContext context = newWatchTestContext()) { + AtomicInteger started = new AtomicInteger(); + context.client.listen("key", response -> { }, throwable -> { }, + started::incrementAndGet); + // Asking PD for a watch is not a live watch yet + assertThat(started.get()).isZero(); + + StreamObserver observer = context.client.call(0).observer; + observer.onNext(startingResponse(7L)); + assertThat(started.get()).isEqualTo(1); + + observer.onNext(WatchResponse.newBuilder().setState(WatchState.Alive).build()); + observer.onNext(eventResponse(7L)); + assertThat(started.get()).isEqualTo(1); + } + } + + @Test + public void testWatchStartedCallbackRunsAgainAfterLeaderChange() throws Exception { + try (WatchTestContext context = newWatchTestContext()) { + AtomicInteger started = new AtomicInteger(); + context.client.listen("key", response -> { }, throwable -> { }, + started::incrementAndGet); + StreamObserver observer = context.client.call(0).observer; + observer.onNext(startingResponse(7L)); + + observer.onNext( + WatchResponse.newBuilder().setState(WatchState.Leader_Changed).build()); + // The watch is down until the new leader answers + assertThat(started.get()).isEqualTo(1); + context.runNextReconnect(); + assertThat(started.get()).isEqualTo(1); + + context.client.call(1).observer.onNext(startingResponse(8L)); + assertThat(started.get()).isEqualTo(2); + } + } + + @Test + public void testWatchStartedCallbackRunsAgainAfterStreamError() throws Exception { + try (WatchTestContext context = newWatchTestContext()) { + AtomicInteger started = new AtomicInteger(); + context.client.listen("key", response -> { }, throwable -> { }, + started::incrementAndGet); + context.client.call(0).observer.onNext(startingResponse(7L)); + + context.client.call(0).observer.onError(Status.UNAVAILABLE.asRuntimeException()); + context.runNextReconnect(); + assertThat(started.get()).isEqualTo(1); + + context.client.call(1).observer.onNext(startingResponse(8L)); + assertThat(started.get()).isEqualTo(2); + } + } + + @Test + public void testWatchStartedCallbackRunsBeforeEventsOfTheNewWatch() throws Exception { + try (WatchTestContext context = newWatchTestContext()) { + List order = new CopyOnWriteArrayList<>(); + context.client.listen("key", response -> order.add("event"), throwable -> { }, + () -> order.add("started")); + + // The first frame of a watch may already carry events + context.client.call(0).observer.onNext(eventResponse(7L)); + context.client.call(0).observer.onNext(eventResponse(7L)); + + assertThat(order).containsExactly("started", "event", "event"); + } + } + + @Test + public void testWatchStartedCallbackIgnoresFramesOfAReplacedWatch() throws Exception { + try (WatchTestContext context = newWatchTestContext()) { + AtomicInteger started = new AtomicInteger(); + context.client.listen("key", response -> { }, throwable -> { }, + started::incrementAndGet); + StreamObserver replaced = context.client.call(0).observer; + replaced.onError(Status.UNAVAILABLE.asRuntimeException()); + context.runNextReconnect(); + + // A late frame of the old stream says nothing about the new watch + replaced.onNext(startingResponse(7L)); + assertThat(started.get()).isZero(); + + context.client.call(1).observer.onNext(startingResponse(8L)); + assertThat(started.get()).isEqualTo(1); + } + } + + @Test + public void testFailingWatchStartedCallbackKeepsTheWatch() throws Exception { + try (WatchTestContext context = newWatchTestContext()) { + AtomicInteger events = new AtomicInteger(); + context.client.listen("key", response -> events.incrementAndGet(), + throwable -> { }, + () -> { + throw new IllegalStateException("callback failed"); + }); + + context.client.call(0).observer.onNext(startingResponse(7L)); + context.client.call(0).observer.onNext(eventResponse(7L)); + + assertThat(events.get()).isEqualTo(1); + assertThat(context.reconnectTasks).isEmpty(); + } + } + + @Test + public void testListenWithoutWatchStartedCallbackStillWorks() throws Exception { + try (WatchTestContext context = newWatchTestContext()) { + AtomicInteger events = new AtomicInteger(); + context.client.listen("key", response -> events.incrementAndGet()); + + context.client.call(0).observer.onNext(startingResponse(7L)); + context.client.call(0).observer.onNext(eventResponse(7L)); + + assertThat(events.get()).isEqualTo(1); + } + } + @Test public void testFirstFrameDuringTimeoutTransitionPreventsReconnect() throws Exception { String loggerName = KvClient.class.getName(); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java index d1eb3abcae..6916d76435 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java @@ -24,7 +24,15 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.BiConsumer; import java.util.function.Consumer; import org.apache.hugegraph.HugeGraphParams; @@ -80,6 +88,19 @@ public class CachedSchemaTransactionV2 extends SchemaTransactionV2 { private static final ConcurrentMap SCHEMA_CACHE_EVENT_LISTENERS = new ConcurrentHashMap<>(); + /** + * How a clear event is sent to the other Servers, given the graph space + * and the graph. It throws if the event could not be sent. It is a field, + * not a direct call, so tests can observe a publish and make one fail + * without a PD. + */ + private static volatile BiConsumer clearEventPublisher = + (graphSpace, graph) -> MetaManager.instance().notifySchemaCacheClear( + graphSpace, graph, SCHEMA_CACHE_CLEAR_SOURCE); + + private static final PendingClearEvents PENDING_CLEAR_EVENTS = + new PendingClearEvents(); + private CacheListenerHolder holder; private final Cache idCache; private final Cache nameCache; @@ -135,24 +156,69 @@ private static String cacheName(String prefix, String spaceGraphName) { private static void clearSchemaCache(String spaceGraphName) { Map> caches = CacheManager.instance().caches(); - - // Clear name cache first so the (name -> id -> object) lookup path - // fails fast instead of returning a stale object backed by an - // already-empty id cache during the TOCTOU window. Cache nameCache = caches.get(cacheName(NAME_CACHE_PREFIX, spaceGraphName)); - if (nameCache != null) { - nameCache.clear(); - } - Cache idCache = caches.get(cacheName(ID_CACHE_PREFIX, spaceGraphName)); - if (idCache != null) { - SchemaCaches arrayCaches = idCache.attachment(); - if (arrayCaches != null) { - arrayCaches.clear(); + SchemaCaches arrayCaches = idCache == null ? + null : idCache.attachment(); + clearCaches(nameCache, arrayCaches, idCache); + } + + /** + * Clear every V2 schema cache of this JVM. Used when the clear-event watch + * becomes live: whatever was published while it was down never arrived. + */ + private static void clearAllSchemaCaches() { + String prefix = ID_CACHE_PREFIX + "-"; + List names = new ArrayList<>(CacheManager.instance() + .caches().keySet()); + for (String name : names) { + if (name.startsWith(prefix)) { + clearSchemaCache(name.substring(prefix.length())); } - idCache.clear(); + } + } + + /** + * Drop everything cached for one graph and advance its generation, so a + * read or a write-through that began before this point can't put its + * value into the cache afterwards. + */ + private static void clearCaches(Cache nameCache, + SchemaCaches arrayCaches, + Cache idCache) { + if (arrayCaches == null) { + /* + * No transaction has attached the guard yet, so nothing reads or + * refills these caches. + */ + if (nameCache != null) { + nameCache.clear(); + } + if (idCache != null) { + idCache.clear(); + } + return; + } + Lock lock = arrayCaches.writeLock(); + lock.lock(); + try { + arrayCaches.advanceGeneration(); + /* + * Clear name cache first so the (name -> id -> object) lookup + * path of a reader that takes no lock fails fast instead of + * returning a stale object backed by an already-empty id cache. + */ + if (nameCache != null) { + nameCache.clear(); + } + arrayCaches.clear(); + if (idCache != null) { + idCache.clear(); + } + } finally { + lock.unlock(); } } @@ -183,21 +249,7 @@ private void listenChanges() { event.checkArgs(String.class, HugeType.class, Id.class); HugeType type = (HugeType) args[1]; Id id = (Id) args[2]; - this.arrayCaches.remove(type, id); - - id = generateId(type, id); - Object value = this.idCache.get(id); - if (value != null) { - // Invalidate id cache - this.idCache.invalidate(id); - - // Invalidate name cache - SchemaElement schema = (SchemaElement) value; - Id prefixedName = generateId(schema.type(), - schema.name()); - this.nameCache.invalidate(prefixedName); - } - this.resetCachedAll(type); + this.invalidateCache(type, id, true); return true; } else if (Cache.ACTION_CLEAR.equals(args[0])) { event.checkArgs(String.class, HugeType.class); @@ -208,9 +260,11 @@ private void listenChanges() { }; EventHub schemaEventHub = this.graphParams().schemaEventHub(); listenSchemaCacheClear(); + boolean[] opened = {false}; CacheListenerHolder acquired = SCHEMA_CACHE_EVENT_LISTENERS.compute( this.graphParams().spaceGraphName(), (key, existing) -> { if (existing == null || existing.hub != schemaEventHub) { + opened[0] = true; if (existing != null) { existing.close(); } @@ -229,6 +283,15 @@ private void listenChanges() { }); this.holder = acquired; this.cacheEventListener = acquired.listener; + if (opened[0]) { + /* + * This JVM just opened the graph. A previous run of this Server + * may have stopped between a schema write and its clear event, + * and the set of failed publishes it kept in memory is gone, so + * tell the other Servers once to drop what they hold. + */ + this.publishClearEvent(); + } } private static void listenSchemaCacheClear() { @@ -238,7 +301,8 @@ private static void listenSchemaCacheClear() { } try { MetaManager.instance().listenSchemaCacheClear( - CachedSchemaTransactionV2::handleSchemaCacheClearEvent); + CachedSchemaTransactionV2::handleSchemaCacheClearEvent, + CachedSchemaTransactionV2::handleSchemaCacheWatchStarted); // Set AFTER the underlying watch is live so a concurrent // caller that observes the flag is guaranteed an active // subscription, and a failure leaves the flag false so the @@ -278,15 +342,24 @@ static void handleSchemaCacheClearEvent(T response) { } } + /** + * Runs each time the clear-event watch becomes live: when it first starts + * and whenever it starts again after a disconnect or a PD leader change. + * Events published while it was down were not delivered and are not + * replayed, so every cache of this JVM is dropped, and clear events of + * this Server that failed to publish are sent again. + */ + static void handleSchemaCacheWatchStarted() { + LOG.info("Schema cache clear watch is live, clear all V2 schema caches"); + clearAllSchemaCaches(); + PENDING_CLEAR_EVENTS.retryNow(); + } + public void clearCache(boolean notify) { - // Same TOCTOU ordering as clearSchemaCache(String): clear nameCache - // first, then the array attachment, then idCache last. - this.nameCache.clear(); - this.arrayCaches.clear(); - this.idCache.clear(); + clearCaches(this.nameCache, this.arrayCaches, this.idCache); if (notify) { - this.maybeNotifySchemaCacheClear(); + this.publishClearEvent(); } } @@ -331,7 +404,26 @@ private void resetCachedAll(HugeType type) { this.cachedTypes().put(type, false); } - private void invalidateCache(HugeType type, Id id) { + /** + * Remove one element from the caches and advance the generation, so a + * read that fetched the element before this point can't cache it again. + */ + private void invalidateCache(HugeType type, Id id, boolean resetCachedAll) { + Lock lock = this.arrayCaches.writeLock(); + lock.lock(); + try { + this.arrayCaches.advanceGeneration(); + this.removeFromCaches(type, id, null); + if (resetCachedAll) { + this.resetCachedAll(type); + } + } finally { + lock.unlock(); + } + } + + // The caller holds the write lock + private void removeFromCaches(HugeType type, Id id, String name) { // remove from id cache and name cache Id prefixedId = generateId(type, id); Object value = this.idCache.get(prefixedId); @@ -342,33 +434,16 @@ private void invalidateCache(HugeType type, Id id) { Id prefixedName = generateId(schema.type(), schema.name()); this.nameCache.invalidate(prefixedName); } + if (name != null) { + this.nameCache.invalidate(generateId(type, name)); + } // remove from optimized array cache this.arrayCaches.remove(type, id); } - @Override - protected void updateSchema(SchemaElement schema, - Consumer updateCallback) { - super.updateSchema(schema, updateCallback); - - this.updateCache(schema); - // Status transitions are internal bookkeeping; notifying here causes a - // broadcast storm for every updateSchemaStatus() call from background jobs. - } - - @Override - protected void addSchema(SchemaElement schema) { - super.addSchema(schema); - - this.updateCache(schema); - - // Schema additions must always propagate to remote nodes regardless - // of TASK_SYNC_DELETION (which only gates removal flows). - this.notifySchemaCacheClear(); - } - - private void updateCache(SchemaElement schema) { + // The caller holds the write lock + private void putIntoCaches(SchemaElement schema) { this.resetCachedAllIfReachedCapacity(); // update id cache @@ -383,34 +458,125 @@ private void updateCache(SchemaElement schema) { this.arrayCaches.updateIfNeeded(schema); } + /** + * Cache an element read from the store, unless the caches were cleared or + * the element was written since {@code generation} was read. The value + * may then be older than what the store holds now, and nothing would + * correct it later because schema caches don't expire. + */ + private void cacheIfCurrent(SchemaElement schema, long generation) { + Lock lock = this.arrayCaches.writeLock(); + lock.lock(); + try { + if (this.arrayCaches.generation() == generation) { + this.putIntoCaches(schema); + } + } finally { + lock.unlock(); + } + } + + // Copy an id-cache hit into the array cache, under the same condition + private void promoteIfCurrent(SchemaElement schema, long generation) { + Lock lock = this.arrayCaches.writeLock(); + lock.lock(); + try { + if (this.arrayCaches.generation() == generation) { + this.arrayCaches.updateIfNeeded(schema); + } + } finally { + lock.unlock(); + } + } + + /** + * Cache an element this transaction just wrote to the store. + * + * @param generation read before the write was sent + */ + private void cacheWritten(SchemaElement schema, long generation) { + Lock lock = this.arrayCaches.writeLock(); + lock.lock(); + try { + boolean current = this.arrayCaches.generation() == generation; + /* + * The write replaced the stored element, so whatever a concurrent + * read fetched before now is old: don't let it be cached. + */ + this.arrayCaches.advanceGeneration(); + if (current) { + this.putIntoCaches(schema); + } else { + /* + * The caches were cleared while the write was on its way, so + * another Server may have replaced the element after this + * write. Cache nothing and let the next read ask the store. + */ + this.removeFromCaches(schema.type(), schema.id(), schema.name()); + this.resetCachedAll(schema.type()); + } + } finally { + lock.unlock(); + } + } + @Override - public void removeSchema(SchemaElement schema) { - super.removeSchema(schema); + protected void updateSchema(SchemaElement schema, + Consumer updateCallback) { + long generation = this.arrayCaches.generation(); + super.updateSchema(schema, updateCallback); - this.invalidateCache(schema.type(), schema.id()); + this.cacheWritten(schema, generation); - this.maybeNotifySchemaCacheClear(); + /* + * An update must reach the other Servers like an add does: they keep + * the whole element until told otherwise, and write it back whole on + * their next update of it. That includes status changes made by + * background jobs. + */ + this.publishClearEvent(); } - private void maybeNotifySchemaCacheClear() { - // Only suppress notifications for removal tasks when - // TASK_SYNC_DELETION=true: the caller propagates cache invalidation - // synchronously, so the meta-event broadcast would be redundant. - if (!this.graph().option(CoreOptions.TASK_SYNC_DELETION)) { - this.notifySchemaCacheClear(); - } + @Override + protected void addSchema(SchemaElement schema) { + long generation = this.arrayCaches.generation(); + super.addSchema(schema); + + this.cacheWritten(schema, generation); + + this.publishClearEvent(); } - private void notifySchemaCacheClear() { - MetaManager.instance() - .notifySchemaCacheClear(this.graph().graphSpace(), - this.graph().name(), - SCHEMA_CACHE_CLEAR_SOURCE); + @Override + public void removeSchema(SchemaElement schema) { + super.removeSchema(schema); + + this.invalidateCache(schema.type(), schema.id(), false); + + /* + * Publish whatever task.sync_deletion says: that option only makes + * the caller wait for the local task, it never reached another JVM. + */ + this.publishClearEvent(); + } + + /** + * Tell the other Servers to drop their schema caches of this graph. It is + * called after the schema write is committed, so it never fails the + * caller: a failed publish is kept and sent again, see + * {@link PendingClearEvents}. + */ + private void publishClearEvent() { + PENDING_CLEAR_EVENTS.publish(this.graph().graphSpace(), + this.graph().name()); } @Override @SuppressWarnings("unchecked") protected T getSchema(HugeType type, Id id) { + // Read it before any cache or store access, see SchemaCaches + long generation = this.arrayCaches.generation(); + // try get from optimized array cache if (id.number() && id.asLong() > 0L) { SchemaElement value = this.arrayCaches.get(type, id); @@ -424,19 +590,13 @@ protected T getSchema(HugeType type, Id id) { if (value == null) { value = super.getSchema(type, id); if (value != null) { - this.resetCachedAllIfReachedCapacity(); - - this.idCache.update(prefixedId, value); - - SchemaElement schema = (SchemaElement) value; - Id prefixedName = generateId(schema.type(), schema.name()); - this.nameCache.update(prefixedName, schema); + this.cacheIfCurrent((SchemaElement) value, generation); } + } else if (this.arrayCaches.accepts(id)) { + // update optimized array cache + this.promoteIfCurrent((SchemaElement) value, generation); } - // update optimized array cache - this.arrayCaches.updateIfNeeded((SchemaElement) value); - return (T) value; } @@ -450,30 +610,64 @@ protected T getSchema(HugeType type, value = super.getSchema(type, name); if (value != null) { // Note: reload all schema if the cache is inconsistent with storage layer - this.clearCache(false); - this.loadAllSchema(); + this.reloadAllSchema(prefixedName); } } return (T) value; } + /** + * Clear and load the whole schema after a name was found in the store but + * not in the cache. One reload runs at a time per graph: a lookup that + * waited finds its name cached by the reload it waited for. Without that, + * every concurrent miss would clear the cache again, which also advances + * the generation and throws away what the other reloads are loading. + */ + private void reloadAllSchema(Id missedName) { + synchronized (this.arrayCaches.reloadLock()) { + if (this.nameCache.get(missedName) != null) { + return; + } + this.clearCache(false); + this.loadAllSchema(); + } + } + @Override protected List getAllSchema(HugeType type) { - Boolean cachedAll = this.cachedTypes().getOrDefault(type, false); - List results; - if (cachedAll) { - results = new ArrayList<>(); - // Get from cache - this.idCache.traverse(value -> { - @SuppressWarnings("unchecked") - T schema = (T) value; - if (schema.type() == type) { - results.add(schema); - } - }); - return results; - } else { - results = super.getAllSchema(type); + long generation; + /* + * A clear can't run while the list is read from the cache, so the + * result is the complete list or the store is asked, never a part. + */ + Lock readLock = this.arrayCaches.readLock(); + readLock.lock(); + try { + generation = this.arrayCaches.generation(); + if (this.cachedTypes().getOrDefault(type, false)) { + List results = new ArrayList<>(); + // Get from cache + this.idCache.traverse(value -> { + @SuppressWarnings("unchecked") + T schema = (T) value; + if (schema.type() == type) { + results.add(schema); + } + }); + return results; + } + } finally { + readLock.unlock(); + } + + List results = super.getAllSchema(type); + Lock writeLock = this.arrayCaches.writeLock(); + writeLock.lock(); + try { + if (this.arrayCaches.generation() != generation) { + // Cleared or written meanwhile: the list may be old already + return results; + } long free = this.idCache.capacity() - this.idCache.size(); if (results.size() <= free) { // Update cache @@ -486,8 +680,10 @@ protected List getAllSchema(HugeType type) { } this.cachedTypes().putIfAbsent(type, true); } - return results; + } finally { + writeLock.unlock(); } + return results; } private void loadAllSchema() { @@ -502,11 +698,134 @@ public void clear() { // Clear schema info firstly super.clear(); this.clearCache(false); - this.notifySchemaCacheClear(); + this.publishClearEvent(); + } + + /** + * Clear events of this Server that could not be published. + *

+ * A clear event is sent after its schema write is committed, so a failed + * publish must not fail the write. The graph is kept here instead and the + * event is sent again: after a delay that doubles up to a limit, and at + * once when the clear-event watch becomes live. One event says "drop your + * caches of this graph", so one successful publish covers every earlier + * failure for that graph. + *

+ * The set lives in memory. If the Server stops before a retry succeeds, + * the publish it does on the next graph open covers it. If the Server + * never returns, the other Servers keep what they cached until the next + * event for that graph or their next watch restart. + */ + static final class PendingClearEvents { + + private static final long RETRY_MIN_DELAY = 1000L; + private static final long RETRY_MAX_DELAY = 30000L; + + private final ConcurrentMap pending = + new ConcurrentHashMap<>(); + private final AtomicLong failures = new AtomicLong(); + + // Guarded by this + private ScheduledExecutorService executor; + private ScheduledFuture retry; + private long retryDelay = RETRY_MIN_DELAY; + + void publish(String graphSpace, String graph) { + try { + clearEventPublisher.accept(graphSpace, graph); + } catch (RuntimeException e) { + LOG.warn("Failed to publish the schema cache clear event of " + + "graph '{}/{}', it will be sent again", + graphSpace, graph, e); + /* + * The number tells a retry whether the graph failed again + * after the publish the retry is about to count as done. + */ + this.pending.put(new GraphRef(graphSpace, graph), + this.failures.incrementAndGet()); + this.scheduleRetry(false); + } + } + + void retryNow() { + this.scheduleRetry(true); + } + + int size() { + return this.pending.size(); + } + + private synchronized void scheduleRetry(boolean now) { + if (this.pending.isEmpty()) { + return; + } + if (this.retry != null && !this.retry.isDone()) { + if (!now) { + return; + } + this.retry.cancel(false); + } + if (this.executor == null) { + this.executor = Executors.newSingleThreadScheduledExecutor(r -> { + Thread thread = new Thread(r, "schema-cache-clear-retry"); + thread.setDaemon(true); + return thread; + }); + } + long delay = now ? 0L : this.retryDelay; + this.retry = this.executor.schedule(this::retryPending, delay, + TimeUnit.MILLISECONDS); + } + + private void retryPending() { + List> entries = + new ArrayList<>(this.pending.entrySet()); + for (Map.Entry entry : entries) { + GraphRef graph = entry.getKey(); + try { + clearEventPublisher.accept(graph.graphSpace(), + graph.graph()); + // Keep it if the graph failed again in the meantime + this.pending.remove(graph, entry.getValue()); + } catch (RuntimeException e) { + LOG.warn("Failed again to publish the schema cache " + + "clear event of graph '{}/{}': {}", + graph.graphSpace(), graph.graph(), e.toString()); + } + } + synchronized (this) { + this.retry = null; + if (this.pending.isEmpty()) { + this.retryDelay = RETRY_MIN_DELAY; + return; + } + this.retryDelay = Math.min(this.retryDelay * 2L, + RETRY_MAX_DELAY); + this.scheduleRetry(false); + } + } } + private record GraphRef(String graphSpace, String graph) { + } + + /** + * The per-graph state shared by every transaction of this JVM: the array + * caches, and the guard that keeps an old value out of all the caches. + *

+ * Schema caches never expire, so a value cached after it was replaced in + * the store stays wrong until the next clear. Every clear, removal and + * local write therefore advances {@link #generation} under the write + * lock. A reader notes the generation before it asks the store and caches + * what it got only if the generation is still the same, checked under the + * same lock. The store is never called with a lock held. + */ private static final class SchemaCaches { + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final Object reloadLock = new Object(); + private volatile long generation; + private final int size; private final IntObjectMap pks; @@ -528,6 +847,36 @@ public SchemaCaches(int size) { this.cachedTypes = new CachedTypes(); } + long generation() { + return this.generation; + } + + // The caller holds the write lock + void advanceGeneration() { + this.generation++; + } + + Lock readLock() { + return this.lock.readLock(); + } + + Lock writeLock() { + return this.lock.writeLock(); + } + + Object reloadLock() { + return this.reloadLock; + } + + // Whether the array cache has a slot for this id + boolean accepts(Id id) { + if (!id.number()) { + return false; + } + long longId = id.asLong(); + return longId > 0L && longId < this.size; + } + public void updateIfNeeded(V schema) { if (schema == null) { return; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaDriver.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaDriver.java index 1409adb47c..31b1cf754a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaDriver.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaDriver.java @@ -37,6 +37,20 @@ public interface MetaDriver { void listen(String key, Consumer consumer); + /** + * Listen on a key and get told each time the watch becomes live: when it + * first starts and whenever it starts again after a disconnect. Events + * committed while a watch is down are not replayed, so {@code onStarted} + * is where a listener rebuilds whatever it derives from them. + *

+ * A driver that can't report this falls back to a plain listen and never + * calls {@code onStarted}. + */ + default void listen(String key, Consumer consumer, + Runnable onStarted) { + this.listen(key, consumer); + } + void listenPrefix(String prefix, Consumer consumer); List extractValuesFromResponse(T response); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java index 0bc917d78d..19c2bff468 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/MetaManager.java @@ -280,6 +280,16 @@ public void listenSchemaCacheClear(Consumer consumer) { this.graphMetaManager.listenSchemaCacheClear(consumer); } + /** + * Listen for schema cache clear events. {@code onStarted} runs each time + * the watch becomes live, see + * {@link MetaDriver#listen(String, Consumer, Runnable)}. + */ + public void listenSchemaCacheClear(Consumer consumer, + Runnable onStarted) { + this.graphMetaManager.listenSchemaCacheClear(consumer, onStarted); + } + public void listenGraphCacheClear(Consumer consumer) { this.graphMetaManager.listenGraphCacheClear(consumer); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/PdMetaDriver.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/PdMetaDriver.java index c744e02e8e..f3f4356f01 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/PdMetaDriver.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/PdMetaDriver.java @@ -116,6 +116,17 @@ public void listen(String key, Consumer consumer) { } } + @Override + public void listen(String key, Consumer consumer, + Runnable onStarted) { + try { + this.client.listen(key, (Consumer) consumer, + throwable -> { }, onStarted); + } catch (PDException e) { + throw new HugeException("Failed to listen '%s' to pd", e, key); + } + } + @Override public void listenPrefix(String prefix, Consumer consumer) { try { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/GraphMetaManager.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/GraphMetaManager.java index 52b2c3946e..1b7f2c5598 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/GraphMetaManager.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/managers/GraphMetaManager.java @@ -185,6 +185,12 @@ public void listenSchemaCacheClear(Consumer consumer) { this.listen(this.schemaCacheClearKey(), consumer); } + public void listenSchemaCacheClear(Consumer consumer, + Runnable onStarted) { + this.metaDriver.listen(this.schemaCacheClearKey(), consumer, + onStarted); + } + public void listenGraphCacheClear(Consumer consumer) { this.listen(this.graphCacheClearKey(), consumer); } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index e3801ba49a..01f3e2c2a5 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -69,6 +69,7 @@ import org.apache.hugegraph.unit.cache.CacheTest; import org.apache.hugegraph.unit.cache.CachedGraphTransactionTest; import org.apache.hugegraph.unit.cache.CachedSchemaTransactionTest; +import org.apache.hugegraph.unit.cache.CachedSchemaTransactionV2Test; import org.apache.hugegraph.unit.cache.RamTableTest; import org.apache.hugegraph.unit.cache.RequestCacheLifetimeTest; import org.apache.hugegraph.unit.cmd.InitStoreConfigTest; @@ -203,6 +204,7 @@ CacheTest.OffheapCacheTest.class, CacheTest.LevelCacheTest.class, CachedSchemaTransactionTest.class, + CachedSchemaTransactionV2Test.class, MetaManagerSchemaCacheClearEventTest.class, EtcdMetaDriverTest.class, CachedGraphTransactionTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java index b019ae16b6..53f10f54bb 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java @@ -866,10 +866,15 @@ public void testListenSchemaCacheClearRegistersOnlyOnce() throws Exception { ArgumentCaptor captor = ArgumentCaptor.forClass(Consumer.class); + ArgumentCaptor startedCaptor = + ArgumentCaptor.forClass(Runnable.class); Mockito.verify(mockDriver, Mockito.times(1)) - .listen(Mockito.anyString(), captor.capture()); + .listen(Mockito.anyString(), captor.capture(), + startedCaptor.capture()); Assert.assertNotNull("registered consumer must not be null", captor.getValue()); + Assert.assertNotNull("the watch-started callback must be registered", + startedCaptor.getValue()); Assert.assertTrue("flag must be set after successful registration", flag.get()); } finally { @@ -912,8 +917,11 @@ public void testListenSchemaCacheClearEndToEnd() throws Exception { ArgumentCaptor captor = ArgumentCaptor.forClass(Consumer.class); + ArgumentCaptor startedCaptor = + ArgumentCaptor.forClass(Runnable.class); Mockito.verify(mockDriver) - .listen(Mockito.anyString(), captor.capture()); + .listen(Mockito.anyString(), captor.capture(), + startedCaptor.capture()); // Simulate the meta server publishing a schema-cache-clear event: // invoke the consumer captured above with a synthetic response. @@ -921,6 +929,15 @@ public void testListenSchemaCacheClearEndToEnd() throws Exception { Assert.assertEquals(0L, idCache.size()); Assert.assertEquals(0L, nameCache.size()); + + // Simulate the watch starting again after a disconnect: events + // published meanwhile never arrived, so the caches are dropped. + idCache.update(IdGenerator.of(1), "v"); + nameCache.update(IdGenerator.of("n"), "v"); + startedCaptor.getValue().run(); + + Assert.assertEquals(0L, idCache.size()); + Assert.assertEquals(0L, nameCache.size()); } finally { flag.set(previousFlag); swapMetaDriver(originalDriver); @@ -1144,7 +1161,8 @@ public void testV2WarmRequestsRetainSchemaWithoutMetadataReadsAndReceiveInvalida storeHub.listen(EventHub.ANY_EVENT, listener); } } - Mockito.verify(driver).put(Mockito.anyString(), Mockito.anyString()); + // One clear event when this JVM opened the graph, one for the Store clear + Mockito.verify(driver, Mockito.times(2)).put(Mockito.anyString(), Mockito.anyString()); CachedSchemaTransactionV2 afterStoreEvent = new CachedSchemaTransactionV2(driver, "test", requestParams); requests.add(afterStoreEvent); Assert.assertEquals("store-cleared", afterStoreEvent.getVertexLabel(original.id()).name()); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java new file mode 100644 index 0000000000..4f71b35e07 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java @@ -0,0 +1,1002 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.unit.cache; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.function.BiConsumer; +import java.util.function.BiPredicate; + +import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.HugeGraphParams; +import org.apache.hugegraph.backend.cache.Cache; +import org.apache.hugegraph.backend.cache.CacheManager; +import org.apache.hugegraph.backend.cache.CachedSchemaTransactionV2; +import org.apache.hugegraph.backend.store.BackendStore; +import org.apache.hugegraph.backend.store.BackendStoreProvider; +import org.apache.hugegraph.config.CoreOptions; +import org.apache.hugegraph.event.Event; +import org.apache.hugegraph.event.EventHub; +import org.apache.hugegraph.event.EventListener; +import org.apache.hugegraph.id.Id; +import org.apache.hugegraph.id.IdGenerator; +import org.apache.hugegraph.meta.PdMetaDriver; +import org.apache.hugegraph.meta.managers.SchemaMetaManager; +import org.apache.hugegraph.struct.schema.PropertyKey; +import org.apache.hugegraph.struct.schema.SchemaElement; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Whitebox; +import org.apache.hugegraph.type.HugeType; +import org.apache.hugegraph.type.define.SchemaStatus; +import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.hugegraph.unit.FakeObjects; +import org.apache.hugegraph.util.Events; +import org.apache.hugegraph.util.LockUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +/** + * Tests of the V2 schema cache against an in-memory store that stands in for + * PD. The store can stop a call after it read or wrote its data and before it + * returns, which is how each test forces the order of a read, a write and a + * clear. "Another Server" is a second schema manager that writes to the same + * store without touching the cache under test, followed by the clear this + * Server would run when that Server's event arrives. + */ +public class CachedSchemaTransactionV2Test extends BaseUnitTest { + + private static final String GRAPH_SPACE = "DEFAULT"; + private static final String CLUSTER = "hg"; + private static final String VERSION = "version"; + private static final long WAIT_SECONDS = 20L; + private static final AtomicInteger GRAPHS = new AtomicInteger(); + + private String graphName; + private String spaceGraph; + private FakeObjects objects; + private HugeGraphParams params; + private BackendStoreProvider provider; + private EventHub schemaEventHub; + private FakeStore store; + private SchemaMetaManager otherServer; + private RecordingPublisher publisher; + private Object originalPublisher; + private boolean originalRegistered; + private ExecutorService executor; + private CachedSchemaTransactionV2 tx; + + @Before + public void setup() throws Exception { + this.graphName = "v2cache" + GRAPHS.incrementAndGet(); + this.spaceGraph = GRAPH_SPACE + "-" + this.graphName; + this.objects = new FakeObjects(this.graphName); + HugeGraph graph = this.objects.graph(); + Mockito.doReturn(GRAPH_SPACE).when(graph).graphSpace(); + Mockito.doReturn(this.spaceGraph).when(graph).spaceGraphName(); + + this.provider = Mockito.mock(BackendStoreProvider.class); + BackendStore backendStore = Mockito.mock(BackendStore.class); + Mockito.doReturn(this.provider).when(backendStore).provider(); + this.schemaEventHub = new EventHub("schema-" + this.graphName); + + this.params = Mockito.mock(HugeGraphParams.class); + Mockito.doReturn(graph).when(this.params).graph(); + Mockito.doReturn(this.graphName).when(this.params).name(); + Mockito.doReturn(this.spaceGraph).when(this.params).spaceGraphName(); + Mockito.doReturn(graph.configuration()).when(this.params).configuration(); + Mockito.doReturn(this.schemaEventHub).when(this.params).schemaEventHub(); + Mockito.doReturn(backendStore).when(this.params).loadGraphStore(); + + LockUtil.init(this.spaceGraph); + this.store = new FakeStore(); + this.otherServer = new SchemaMetaManager(this.store.driver, CLUSTER, graph); + this.executor = Executors.newCachedThreadPool(); + + // There is no PD: don't register the meta watch, and record publishes + this.originalRegistered = registeredFlag().getAndSet(true); + // Other test classes open V2 graphs without a PD and leave failed publishes + pendingMap().clear(); + this.publisher = new RecordingPublisher(); + this.originalPublisher = Whitebox.getInternalState( + CachedSchemaTransactionV2.class, "clearEventPublisher"); + Whitebox.setInternalState(CachedSchemaTransactionV2.class, + "clearEventPublisher", this.publisher); + + this.tx = new CachedSchemaTransactionV2(this.store.driver, CLUSTER, + this.params); + } + + @After + public void teardown() throws Exception { + this.store.open(); + this.executor.shutdownNow(); + Assert.assertTrue(this.executor.awaitTermination(WAIT_SECONDS, + TimeUnit.SECONDS)); + this.tx.close(); + CachedSchemaTransactionV2.closeGraph(this.params); + pendingMap().clear(); + Whitebox.setInternalState(CachedSchemaTransactionV2.class, + "clearEventPublisher", this.originalPublisher); + registeredFlag().set(this.originalRegistered); + LockUtil.destroy(this.spaceGraph); + } + + // ------------------------------------------------------------ publish + + @Test + public void testGraphOpenPublishesOnce() { + // setup() opened the graph + Assert.assertEquals(1, this.publisher.count(this.graphName)); + + CachedSchemaTransactionV2 second = + new CachedSchemaTransactionV2(this.store.driver, CLUSTER, this.params); + try { + Assert.assertEquals(1, this.publisher.count(this.graphName)); + } finally { + second.close(); + } + } + + @Test + public void testAddPublishes() { + this.publisher.reset(); + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + Assert.assertEquals(1, this.publisher.count(this.graphName)); + } + + @Test + public void testUpdatePublishes() { + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + this.publisher.reset(); + + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + + Assert.assertEquals(1, this.publisher.count(this.graphName)); + Assert.assertEquals(GRAPH_SPACE, this.publisher.events.get(0)[0]); + } + + @Test + public void testStatusChangePublishes() { + PropertyKey pk = this.propertyKey(1, "name", 1); + this.tx.addPropertyKey(pk); + this.publisher.reset(); + + this.tx.updateSchemaStatus(pk, SchemaStatus.INVALID); + + Assert.assertEquals(1, this.publisher.count(this.graphName)); + Assert.assertEquals(SchemaStatus.INVALID, + this.otherServer.getPropertyKey(GRAPH_SPACE, this.graphName, + IdGenerator.of(1)).status()); + } + + @Test + public void testRemovePublishesWhateverSyncDeletionSays() { + for (boolean syncDeletion : new boolean[]{false, true}) { + this.params.configuration().setProperty( + CoreOptions.TASK_SYNC_DELETION.name(), String.valueOf(syncDeletion)); + PropertyKey pk = this.propertyKey(1, "name", 1); + this.tx.addPropertyKey(pk); + this.publisher.reset(); + + this.tx.removeSchema(pk); + + Assert.assertEquals("task.sync_deletion=" + syncDeletion, + 1, this.publisher.count(this.graphName)); + } + } + + @Test + public void testStoreClearAndTruncatePublishButInitDoesNot() { + ArgumentCaptor captor = + ArgumentCaptor.forClass(EventListener.class); + Mockito.verify(this.provider).listen(captor.capture()); + EventListener storeListener = captor.getValue(); + this.params.configuration().setProperty( + CoreOptions.TASK_SYNC_DELETION.name(), "true"); + + for (String event : new String[]{Events.STORE_CLEAR, Events.STORE_TRUNCATE}) { + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + this.publisher.reset(); + long reads = this.store.reads(); + + storeListener.event(new Event(this.provider, event)); + + Assert.assertEquals(event, 1, this.publisher.count(this.graphName)); + // The local cache is dropped too + Assert.assertNotNull(this.tx.getPropertyKey(IdGenerator.of(1))); + Assert.assertEquals(event, reads + 1, this.store.reads()); + } + + this.publisher.reset(); + storeListener.event(new Event(this.provider, Events.STORE_INIT)); + Assert.assertEquals(0, this.publisher.count(this.graphName)); + } + + @Test + public void testSchemaClearPublishes() { + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + this.publisher.reset(); + + this.tx.clear(); + + Assert.assertEquals(1, this.publisher.count(this.graphName)); + Assert.assertNull(this.tx.getPropertyKey(IdGenerator.of(1))); + } + + // ------------------------------------------------------------ failed publish + + @Test + public void testFailedPublishDoesNotFailTheWriteAndIsSentAgain() throws Exception { + this.publisher.reset(); + this.publisher.fail = true; + + // Committed in the store although the event could not be sent + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + Assert.assertNotNull(this.otherServer.getPropertyKey(GRAPH_SPACE, this.graphName, + IdGenerator.of(1))); + Assert.assertEquals(0, this.publisher.count(this.graphName)); + Assert.assertTrue(this.pending()); + + this.publisher.fail = false; + // Sent again by the retry with no further schema change + this.await(() -> this.publisher.count(this.graphName) == 1); + this.await(() -> !this.pending()); + } + + @Test + public void testWatchStartSendsFailedPublishesAtOnce() throws Exception { + this.publisher.reset(); + this.publisher.fail = true; + this.tx.updatePropertyKey(this.propertyKey(1, "name", 1)); + // Let the first retry fail too, so the next one is two seconds away + this.await(() -> this.publisher.attempts.get() >= 2); + + this.publisher.fail = false; + long start = System.nanoTime(); + watchStarted(); + + this.await(() -> this.publisher.count(this.graphName) == 1); + long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + Assert.assertTrue("sent after " + millis + " ms", millis < 1500L); + this.await(() -> !this.pending()); + } + + @Test + public void testFailureDuringARetryIsKept() throws Exception { + this.publisher.reset(); + this.publisher.fail = true; + this.tx.updatePropertyKey(this.propertyKey(1, "name", 1)); + Assert.assertTrue(this.pending()); + + /* + * The retry sends its event, and before it marks the graph as done a + * newer schema change fails to publish. That newer change is not + * covered by the event the retry sent, so the graph must stay pending. + */ + CountDownLatch retryInPublish = new CountDownLatch(1); + CountDownLatch finishRetry = new CountDownLatch(1); + Thread testThread = Thread.currentThread(); + this.publisher.fail = false; + this.publisher.intercept = () -> { + if (Thread.currentThread() == testThread) { + throw new IllegalStateException("PD is unreachable"); + } + retryInPublish.countDown(); + awaitLatch(finishRetry); + }; + watchStarted(); + awaitLatch(retryInPublish); + + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + this.publisher.intercept = null; + this.publisher.fail = true; + finishRetry.countDown(); + + this.await(() -> this.publisher.count(this.graphName) == 1); + Thread.sleep(200L); + Assert.assertTrue(this.pending()); + + this.publisher.fail = false; + this.await(() -> !this.pending()); + Assert.assertEquals(2, this.publisher.count(this.graphName)); + } + + // ------------------------------------------------------------ watch start + + @Test + public void testWatchStartClearsEveryGraph() throws Exception { + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + Assert.assertEquals(1, this.version(this.tx.getPropertyKey(IdGenerator.of(1)))); + Assert.assertEquals(1, this.tx.getPropertyKeys().size()); + // A cache of another graph of this JVM + Cache otherId = CacheManager.instance() + .cache("schema-v2-id-DEFAULT-other", 10L); + Cache otherName = CacheManager.instance() + .cache("schema-v2-name-DEFAULT-other", 10L); + otherId.update(IdGenerator.of(9), "id"); + otherName.update(IdGenerator.of("nine"), "name"); + + // Changed and added by another Server while the watch was down + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 2)); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(2, "age", 1)); + Assert.assertEquals(1, this.version(this.tx.getPropertyKey(IdGenerator.of(1)))); + + watchStarted(); + + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(IdGenerator.of(1)))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + Assert.assertEquals(2, this.tx.getPropertyKeys().size()); + Assert.assertEquals(0L, otherId.size()); + Assert.assertEquals(0L, otherName.size()); + } + + // ------------------------------------------------------------ guard: reads + + @Test + public void testReadByIdStartedBeforeAClearIsNotCached() throws Exception { + Id id = IdGenerator.of(1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + + Gate gate = this.store.stopNext("get", "/ID/1"); + Future read = this.executor.submit(() -> this.tx.getPropertyKey(id)); + gate.awaitReached(); + // The read holds version 1. Another Server replaces it and its event arrives. + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 2)); + this.clearByEvent(); + gate.open(); + Assert.assertEquals(1, this.version(read.get(WAIT_SECONDS, TimeUnit.SECONDS))); + + long reads = this.store.reads(); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertTrue("the old value must not be served from the cache", + this.store.reads() > reads); + // And what is cached now is the new one + reads = this.store.reads(); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(reads, this.store.reads()); + } + + @Test + public void testReadByNameStartedBeforeAClearIsNotCached() throws Exception { + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + + Gate gate = this.store.stopNext("scan", "PROPERTY_KEY"); + Future read = this.executor.submit(() -> this.tx.getPropertyKey("name")); + gate.awaitReached(); + // The reload that follows the name miss has scanned version 1 + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 2)); + this.clearByEvent(); + gate.open(); + read.get(WAIT_SECONDS, TimeUnit.SECONDS); + + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(IdGenerator.of(1)))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKeys().get(0))); + } + + @Test + public void testListingStartedBeforeAClearIsNotCached() throws Exception { + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + + Gate gate = this.store.stopNext("scan", "PROPERTY_KEY"); + Future> listing = this.executor.submit(() -> this.tx.getPropertyKeys()); + gate.awaitReached(); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(2, "age", 1)); + this.clearByEvent(); + gate.open(); + Assert.assertEquals(1, listing.get(WAIT_SECONDS, TimeUnit.SECONDS).size()); + + // The list of one must not be kept as "all property keys" + Assert.assertEquals(2, this.tx.getPropertyKeys().size()); + Assert.assertNotNull(this.tx.getPropertyKey("age")); + } + + @Test + public void testReadStartedBeforeALocalWriteIsNotCached() throws Exception { + Id id = IdGenerator.of(1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + + Gate gate = this.store.stopNext("get", "/ID/1"); + Future read = this.executor.submit(() -> this.tx.getPropertyKey(id)); + gate.awaitReached(); + // This Server itself replaces the element while the read is in flight + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + gate.open(); + Assert.assertEquals(1, this.version(read.get(WAIT_SECONDS, TimeUnit.SECONDS))); + + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + } + + @Test + public void testReadStartedBeforeARemoveIsNotCached() throws Exception { + Id id = IdGenerator.of(1); + PropertyKey pk = this.propertyKey(1, "name", 1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, pk); + + Gate gate = this.store.stopNext("get", "/ID/1"); + Future read = this.executor.submit(() -> this.tx.getPropertyKey(id)); + gate.awaitReached(); + this.tx.removeSchema(pk); + gate.open(); + Assert.assertNotNull(read.get(WAIT_SECONDS, TimeUnit.SECONDS)); + + Assert.assertNull(this.tx.getPropertyKey(id)); + Assert.assertNull(this.tx.getPropertyKey("name")); + Assert.assertEquals(0, this.tx.getPropertyKeys().size()); + } + + // ------------------------------------------------------------ guard: write-through + + @Test + public void testUpdateWriteThroughAfterAClearIsNotCached() throws Exception { + Id id = IdGenerator.of(1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + this.tx.getPropertyKeys(); + + // The write of version 2 reaches the store and its answer is delayed + Gate gate = this.store.stopNext("put", "/NAME/name"); + Future write = this.executor.submit( + () -> this.tx.updatePropertyKey(this.propertyKey(1, "name", 2))); + gate.awaitReached(); + // Another Server writes version 3 after it, and its event arrives first + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 3)); + this.clearByEvent(); + gate.open(); + write.get(WAIT_SECONDS, TimeUnit.SECONDS); + + Assert.assertEquals(3, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(3, this.version(this.tx.getPropertyKey("name"))); + Assert.assertEquals(3, this.version(this.tx.getPropertyKeys().get(0))); + } + + @Test + public void testAddWriteThroughAfterAClearIsNotCached() throws Exception { + Id id = IdGenerator.of(1); + + Gate gate = this.store.stopNext("put", "/NAME/name"); + Future write = this.executor.submit( + () -> this.tx.addPropertyKey(this.propertyKey(1, "name", 1))); + gate.awaitReached(); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 2)); + this.clearByEvent(); + gate.open(); + write.get(WAIT_SECONDS, TimeUnit.SECONDS); + + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + } + + @Test + public void testWriteThroughWithoutAClearIsCached() { + Id id = IdGenerator.of(1); + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + long reads = this.store.reads(); + Assert.assertEquals(1, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(1, this.version(this.tx.getPropertyKey("name"))); + + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + Assert.assertEquals("served by the write-through", reads, this.store.reads()); + } + + // ------------------------------------------------------------ guard: each insertion + + @Test + public void testEveryInsertionChecksTheGeneration() throws Exception { + Id id = IdGenerator.of(1); + PropertyKey old = this.propertyKey(1, "name", 1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 2)); + + long before = this.generation(); + this.clearByEvent(); + Assert.assertTrue(this.generation() > before); + + // A value read from the store before the clear + this.invoke("cacheIfCurrent", new Class[]{SchemaElement.class, long.class}, + old, before); + Assert.assertNull(this.arrayCached(id)); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + + // An id-cache hit taken before the clear, copied to the array cache + this.clearByEvent(); + this.invoke("promoteIfCurrent", new Class[]{SchemaElement.class, long.class}, + old, before); + Assert.assertNull(this.arrayCached(id)); + + // A write sent before the clear + this.invoke("cacheWritten", new Class[]{SchemaElement.class, long.class}, + old, before); + Assert.assertNull(this.arrayCached(id)); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + + // With the current generation each of them caches + this.clearByEvent(); + long current = this.generation(); + PropertyKey fresh = this.propertyKey(1, "name", 2); + this.invoke("cacheIfCurrent", new Class[]{SchemaElement.class, long.class}, + fresh, current); + Assert.assertSame(fresh, this.arrayCached(id)); + long reads = this.store.reads(); + Assert.assertSame(fresh, this.tx.getPropertyKey(id)); + Assert.assertSame(fresh, this.tx.getPropertyKey("name")); + Assert.assertEquals(reads, this.store.reads()); + } + + @Test + public void testClearsRemovesAndWritesAdvanceTheGeneration() { + PropertyKey pk = this.propertyKey(1, "name", 1); + + long generation = this.generation(); + this.tx.addPropertyKey(pk); + Assert.assertTrue("add", this.generation() > generation); + + generation = this.generation(); + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + Assert.assertTrue("update", this.generation() > generation); + + generation = this.generation(); + this.clearByEvent(); + Assert.assertTrue("clear by event", this.generation() > generation); + + generation = this.generation(); + this.tx.clearCache(false); + Assert.assertTrue("local clear", this.generation() > generation); + + generation = this.generation(); + watchStarted(); + Assert.assertTrue("watch start", this.generation() > generation); + + generation = this.generation(); + this.tx.removeSchema(pk); + Assert.assertTrue("remove", this.generation() > generation); + + // A plain read changes nothing + generation = this.generation(); + this.tx.getPropertyKey(IdGenerator.of(1)); + this.tx.getPropertyKeys(); + Assert.assertEquals(generation, this.generation()); + } + + // ------------------------------------------------------------ name miss + + @Test + public void testConcurrentNameMissesReloadOnce() throws Exception { + for (int i = 1; i <= 3; i++) { + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(i, "pk" + i, 1)); + } + int threads = 6; + // Every lookup misses the cache and finds its name in the store first + CountDownLatch missed = new CountDownLatch(threads); + CountDownLatch go = new CountDownLatch(1); + this.store.hook = (operation, key) -> { + if (operation.equals("get") && key.endsWith("/NAME/pk1")) { + missed.countDown(); + awaitLatch(go); + } + }; + List> lookups = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + lookups.add(this.executor.submit(() -> this.tx.getPropertyKey("pk1"))); + } + awaitLatch(missed); + this.store.hook = null; + long scans = this.store.scans.get(); + go.countDown(); + for (Future lookup : lookups) { + Assert.assertNotNull(lookup.get(WAIT_SECONDS, TimeUnit.SECONDS)); + } + + // One reload reads the four schema types, the other lookups reuse it + Assert.assertEquals(4L, this.store.scans.get() - scans); + long reads = this.store.reads(); + Assert.assertNotNull(this.tx.getPropertyKey("pk2")); + Assert.assertNotNull(this.tx.getPropertyKey(IdGenerator.of(3))); + Assert.assertEquals(3, this.tx.getPropertyKeys().size()); + Assert.assertEquals("the cache must stay filled after the misses", + reads, this.store.reads()); + } + + // ------------------------------------------------------------ listing + + @Test + public void testAClearWaitsForAListingReadFromTheCache() throws Exception { + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + this.tx.getPropertyKeys(); + + // Stand in for a listing that is reading the cached list + Lock readLock = (Lock) this.callGuard("readLock"); + readLock.lock(); + Future clear; + try { + clear = this.executor.submit(this::clearByEvent); + try { + clear.get(300L, TimeUnit.MILLISECONDS); + Assert.fail("the clear ran while a listing was reading the cache"); + } catch (java.util.concurrent.TimeoutException expected) { + // The clear waits + } + } finally { + readLock.unlock(); + } + clear.get(WAIT_SECONDS, TimeUnit.SECONDS); + } + + @Test + public void testListingIsNeverPartialWhileClearsRun() throws Exception { + int count = 50; + for (int i = 1; i <= count; i++) { + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(i, "pk" + i, 1)); + } + AtomicBoolean stop = new AtomicBoolean(false); + Future clears = this.executor.submit(() -> { + while (!stop.get()) { + this.clearByEvent(); + } + }); + try { + for (int i = 0; i < 3000; i++) { + Assert.assertEquals("listing " + i, count, this.tx.getPropertyKeys().size()); + } + } finally { + stop.set(true); + } + clears.get(WAIT_SECONDS, TimeUnit.SECONDS); + } + + @Test + public void testReadsAndWritesStayConsistentUnderConcurrentClears() throws Exception { + Id id = IdGenerator.of(1); + this.tx.addPropertyKey(this.propertyKey(1, "name", 0)); + AtomicBoolean stop = new AtomicBoolean(false); + List> noise = new ArrayList<>(); + noise.add(this.executor.submit(() -> { + while (!stop.get()) { + this.clearByEvent(); + } + })); + for (int i = 0; i < 3; i++) { + noise.add(this.executor.submit(() -> { + while (!stop.get()) { + this.tx.getPropertyKey(id); + this.tx.getPropertyKey("name"); + this.tx.getPropertyKeys(); + } + })); + } + try { + for (int version = 1; version <= 300; version++) { + this.tx.updatePropertyKey(this.propertyKey(1, "name", version)); + } + } finally { + stop.set(true); + } + for (Future future : noise) { + future.get(WAIT_SECONDS, TimeUnit.SECONDS); + } + + // Whatever ended up cached, it is the last version written + Assert.assertEquals(300, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(300, this.version(this.tx.getPropertyKey("name"))); + Assert.assertEquals(300, this.version(this.tx.getPropertyKeys().get(0))); + } + + // ------------------------------------------------------------ helpers + + private PropertyKey propertyKey(int id, String name, int version) { + PropertyKey pk = this.objects.newPropertyKey(IdGenerator.of(id), name); + pk.userdata(VERSION, version); + return pk; + } + + private int version(PropertyKey pk) { + Assert.assertNotNull(pk); + return ((Number) pk.userdata().get(VERSION)).intValue(); + } + + /** What this Server does when another Server's clear event arrives. */ + private void clearByEvent() { + Whitebox.invokeStatic(CachedSchemaTransactionV2.class, + new Class[]{String.class}, + "clearSchemaCache", this.spaceGraph); + } + + private static void watchStarted() { + try { + Method method = CachedSchemaTransactionV2.class + .getDeclaredMethod("handleSchemaCacheWatchStarted"); + method.setAccessible(true); + method.invoke(null); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } + + private Object guard() { + Object guard = Whitebox.getInternalState(this.tx, "arrayCaches"); + Assert.assertNotNull(guard); + return guard; + } + + private Object callGuard(String name, Object... args) { + Object guard = this.guard(); + try { + for (Method method : guard.getClass().getDeclaredMethods()) { + if (method.getName().equals(name) && + method.getParameterCount() == args.length) { + method.setAccessible(true); + return method.invoke(guard, args); + } + } + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + throw new AssertionError("No method " + name); + } + + private long generation() { + return (Long) this.callGuard("generation"); + } + + private Object arrayCached(Id id) { + return this.callGuard("get", HugeType.PROPERTY_KEY, id); + } + + private void invoke(String name, Class[] types, Object... args) { + try { + Method method = CachedSchemaTransactionV2.class.getDeclaredMethod(name, types); + method.setAccessible(true); + method.invoke(this.tx, args); + } catch (ReflectiveOperationException e) { + throw new AssertionError(e); + } + } + + private static AtomicBoolean registeredFlag() throws Exception { + Field field = CachedSchemaTransactionV2.class + .getDeclaredField("metaEventListenerRegistered"); + field.setAccessible(true); + return (AtomicBoolean) field.get(null); + } + + private static Map pendingMap() { + Object pending = Whitebox.getInternalState(CachedSchemaTransactionV2.class, + "PENDING_CLEAR_EVENTS"); + return Whitebox.getInternalState(pending, "pending"); + } + + // Whether this graph has a clear event waiting to be published again + private boolean pending() { + for (Object graph : pendingMap().keySet()) { + if (graph.toString().contains(this.graphName + "]")) { + return true; + } + } + return false; + } + + private void await(Condition condition) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(WAIT_SECONDS); + while (!condition.met()) { + Assert.assertTrue("timed out", System.nanoTime() < deadline); + Thread.sleep(10L); + } + } + + private static void awaitLatch(CountDownLatch latch) { + try { + if (!latch.await(WAIT_SECONDS, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting for the other thread"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + @FunctionalInterface + private interface Condition { + + boolean met() throws Exception; + } + + /** Stops one store call after it did its work and before it returns. */ + private static final class Gate { + + private final CountDownLatch reached = new CountDownLatch(1); + private final CountDownLatch opened = new CountDownLatch(1); + + void awaitReached() { + awaitLatch(this.reached); + } + + void open() { + this.opened.countDown(); + } + + void stop() { + this.reached.countDown(); + awaitLatch(this.opened); + } + } + + private static final class RecordingPublisher implements BiConsumer { + + private final List events = new CopyOnWriteArrayList<>(); + private final AtomicInteger attempts = new AtomicInteger(); + private volatile boolean fail; + private volatile Runnable intercept; + + @Override + public void accept(String graphSpace, String graph) { + this.attempts.incrementAndGet(); + // Decided when the call starts, like a request already on its way + boolean failed = this.fail; + Runnable current = this.intercept; + if (current != null) { + current.run(); + } + if (failed) { + throw new IllegalStateException("PD is unreachable"); + } + this.events.add(new String[]{graphSpace, graph}); + } + + int count(String graph) { + int count = 0; + for (String[] event : this.events) { + if (event[1].equals(graph)) { + count++; + } + } + return count; + } + + void reset() { + this.events.clear(); + this.attempts.set(0); + } + } + + /** An in-memory stand-in for the PD key-value store. */ + private static final class FakeStore { + + private final ConcurrentSkipListMap data = + new ConcurrentSkipListMap<>(); + private final AtomicInteger gets = new AtomicInteger(); + private final AtomicInteger scans = new AtomicInteger(); + private final List gates = new CopyOnWriteArrayList<>(); + private final PdMetaDriver driver = Mockito.mock(PdMetaDriver.class); + private volatile StoreHook hook; + + FakeStore() { + Mockito.doAnswer(invocation -> { + String key = invocation.getArgument(0); + this.data.put(key, invocation.getArgument(1)); + this.after("put", key); + return null; + }).when(this.driver).put(Mockito.anyString(), Mockito.anyString()); + Mockito.doAnswer(invocation -> { + String key = invocation.getArgument(0); + String value = this.data.get(key); + this.gets.incrementAndGet(); + this.after("get", key); + return value; + }).when(this.driver).get(Mockito.anyString()); + Mockito.doAnswer(invocation -> { + String key = invocation.getArgument(0); + this.data.remove(key); + this.after("delete", key); + return null; + }).when(this.driver).delete(Mockito.anyString()); + Mockito.doAnswer(invocation -> { + String prefix = invocation.getArgument(0); + this.data.keySet().removeIf(key -> key.startsWith(prefix)); + this.after("delete", prefix); + return null; + }).when(this.driver).deleteWithPrefix(Mockito.anyString()); + Mockito.doAnswer(invocation -> { + String prefix = invocation.getArgument(0); + Map result = new TreeMap<>(); + this.data.forEach((key, value) -> { + if (key.startsWith(prefix)) { + result.put(key, value); + } + }); + this.scans.incrementAndGet(); + this.after("scan", prefix); + return result; + }).when(this.driver).scanWithPrefix(Mockito.anyString()); + } + + long reads() { + return this.gets.get() + this.scans.get(); + } + + /** Stop the next call of this operation whose key contains the text. */ + Gate stopNext(String operation, String keyPart) { + Gate gate = new Gate(); + AtomicBoolean used = new AtomicBoolean(false); + BiPredicate matches = + (op, key) -> op.equals(operation) && key.contains(keyPart); + this.gates.add(gate); + this.hook = (op, key) -> { + if (matches.test(op, key) && used.compareAndSet(false, true)) { + gate.stop(); + } + }; + return gate; + } + + void open() { + this.hook = null; + for (Gate gate : this.gates) { + gate.open(); + } + } + + private void after(String operation, String key) { + StoreHook current = this.hook; + if (current != null) { + current.after(operation, key); + } + } + } + + @FunctionalInterface + private interface StoreHook { + + void after(String operation, String key); + } +} From 2e75022dceb5bfa542a39b32c7e64ca66cdc089d Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 13 Nov 2026 14:17:05 +0530 Subject: [PATCH 2/3] fix(server): publish after a failed schema write and fix cache guard A second pass over the schema cache clear path found four defects. - A schema write is two or three PD puts. One that failed after the first put left the change partly in PD, published nothing and kept the writer's own copy. It now drops that copy, publishes, and still reports the error. The publish also runs if caching the written element fails. - The generation was advanced before the caches were changed. A read running at that moment could note the new generation, take the old value from the id cache and copy it into the array cache afterwards, where it stayed until the next clear. The generation is now advanced as the last step of every change. - A watch start during a running retry scheduled a second retry chain, which doubled the publishes and the backoff. Only the newest retry reschedules now. - After a refused write-through the "whole list is cached" flag of that schema type was never set again, so each listing read PD until the next clear. Tests: one per fix, plus tests for the publish order, the callback passed by PdMetaDriver, the guard across transactions of one graph and local invalidation events. The listing-lock test now holds a real listing, and the concurrency test checks every read while it runs. A V2 graph opened without PD no longer leaves a retry behind that fires in a later test. --- docs/server-runtime.md | 7 + .../apache/hugegraph/ct/env/AbstractEnv.java | 3 +- .../cache/CachedSchemaTransactionV2.java | 136 ++++-- .../cache/CachedSchemaTransactionTest.java | 7 +- .../cache/CachedSchemaTransactionV2Test.java | 449 ++++++++++++++++-- 5 files changed, 531 insertions(+), 71 deletions(-) diff --git a/docs/server-runtime.md b/docs/server-runtime.md index ba92d299b5..5cb0da6f16 100644 --- a/docs/server-runtime.md +++ b/docs/server-runtime.md @@ -838,6 +838,9 @@ What a deployment can rely on once every Server runs this version: removing property keys, labels and index labels, the status changes made by index and label jobs, and clearing or truncating a graph. A removal is published whatever `task.sync_deletion` is set to. +- A schema change that fails after it changed part of what PD stores is published + too. The Server that made it drops its own copy of that element, and the caller + still gets the error. - A Server whose PD watch was down drops all its cached schema when the watch is back, for example after a PD restart or a PD leader change. Events published in between are not replayed, and this is how they are made up for. @@ -848,6 +851,10 @@ What a deployment can rely on once every Server runs this version: Each Server also publishes once when it opens a graph, which covers a Server that stopped between a schema write and its event. +Each event makes the other Servers load the whole schema of that graph again on their +next read by name. Schema changes that arrive back to back therefore slow schema reads +on the other Servers until the burst ends; no request fails because of it. + What it does not provide: - Writes during index creation. A Server learns about a new index when it handles the diff --git a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java index 8e1311a0e5..1bd7246d51 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/env/AbstractEnv.java @@ -147,7 +147,8 @@ private static void sleepBeforeRetry() { } /** - * Restart one PD node on its own data and return without waiting for it. + * Restart one PD node on its own data. It waits for the old process to + * exit and returns without waiting for the new one to be ready. * Restarting the PD leader makes the leader change and every PD watch * start again, which is what a test of that window needs to race. */ diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java index 6916d76435..4ba6a1506a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java @@ -204,7 +204,6 @@ private static void clearCaches(Cache nameCache, Lock lock = arrayCaches.writeLock(); lock.lock(); try { - arrayCaches.advanceGeneration(); /* * Clear name cache first so the (name -> id -> object) lookup * path of a reader that takes no lock fails fast instead of @@ -218,6 +217,8 @@ private static void clearCaches(Cache nameCache, idCache.clear(); } } finally { + // Last, see SchemaCaches + arrayCaches.advanceGeneration(); lock.unlock(); } } @@ -249,7 +250,7 @@ private void listenChanges() { event.checkArgs(String.class, HugeType.class, Id.class); HugeType type = (HugeType) args[1]; Id id = (Id) args[2]; - this.invalidateCache(type, id, true); + this.invalidateCache(type, id, null, true); return true; } else if (Cache.ACTION_CLEAR.equals(args[0])) { event.checkArgs(String.class, HugeType.class); @@ -408,16 +409,18 @@ private void resetCachedAll(HugeType type) { * Remove one element from the caches and advance the generation, so a * read that fetched the element before this point can't cache it again. */ - private void invalidateCache(HugeType type, Id id, boolean resetCachedAll) { + private void invalidateCache(HugeType type, Id id, String name, + boolean resetCachedAll) { Lock lock = this.arrayCaches.writeLock(); lock.lock(); try { - this.arrayCaches.advanceGeneration(); - this.removeFromCaches(type, id, null); + this.removeFromCaches(type, id, name); if (resetCachedAll) { this.resetCachedAll(type); } } finally { + // Last, see SchemaCaches + this.arrayCaches.advanceGeneration(); lock.unlock(); } } @@ -498,13 +501,7 @@ private void cacheWritten(SchemaElement schema, long generation) { Lock lock = this.arrayCaches.writeLock(); lock.lock(); try { - boolean current = this.arrayCaches.generation() == generation; - /* - * The write replaced the stored element, so whatever a concurrent - * read fetched before now is old: don't let it be cached. - */ - this.arrayCaches.advanceGeneration(); - if (current) { + if (this.arrayCaches.generation() == generation) { this.putIntoCaches(schema); } else { /* @@ -516,6 +513,12 @@ private void cacheWritten(SchemaElement schema, long generation) { this.resetCachedAll(schema.type()); } } finally { + /* + * The write replaced the stored element, so whatever a concurrent + * read fetched before now is old: don't let it be cached. Last, + * see SchemaCaches. + */ + this.arrayCaches.advanceGeneration(); lock.unlock(); } } @@ -524,9 +527,12 @@ private void cacheWritten(SchemaElement schema, long generation) { protected void updateSchema(SchemaElement schema, Consumer updateCallback) { long generation = this.arrayCaches.generation(); - super.updateSchema(schema, updateCallback); - - this.cacheWritten(schema, generation); + try { + super.updateSchema(schema, updateCallback); + } catch (Throwable e) { + this.writeFailed(schema); + throw e; + } /* * An update must reach the other Servers like an add does: they keep @@ -534,30 +540,64 @@ protected void updateSchema(SchemaElement schema, * their next update of it. That includes status changes made by * background jobs. */ - this.publishClearEvent(); + try { + this.cacheWritten(schema, generation); + } finally { + this.publishClearEvent(); + } } @Override protected void addSchema(SchemaElement schema) { long generation = this.arrayCaches.generation(); - super.addSchema(schema); - - this.cacheWritten(schema, generation); + try { + super.addSchema(schema); + } catch (Throwable e) { + this.writeFailed(schema); + throw e; + } - this.publishClearEvent(); + try { + this.cacheWritten(schema, generation); + } finally { + this.publishClearEvent(); + } } @Override public void removeSchema(SchemaElement schema) { - super.removeSchema(schema); - - this.invalidateCache(schema.type(), schema.id(), false); + try { + super.removeSchema(schema); + } catch (Throwable e) { + this.writeFailed(schema); + throw e; + } /* * Publish whatever task.sync_deletion says: that option only makes * the caller wait for the local task, it never reached another JVM. */ - this.publishClearEvent(); + try { + this.invalidateCache(schema.type(), schema.id(), schema.name(), + false); + } finally { + this.publishClearEvent(); + } + } + + /** + * A schema write is several puts or deletes in the store, so one that + * threw may have changed the store in part or in full. Treat it like a + * write: drop the element here, so nothing serves a copy the store no + * longer matches, and tell the other Servers to do the same. + */ + private void writeFailed(SchemaElement schema) { + try { + this.invalidateCache(schema.type(), schema.id(), schema.name(), + true); + } finally { + this.publishClearEvent(); + } } /** @@ -678,7 +718,11 @@ protected List getAllSchema(HugeType type) { Id prefixedName = generateId(schema.type(), schema.name()); this.nameCache.update(prefixedName, schema); } - this.cachedTypes().putIfAbsent(type, true); + /* + * The generation is unchanged, so this is the whole list: + * also replace a "not all cached" left by an earlier write + */ + this.cachedTypes().put(type, true); } } finally { writeLock.unlock(); @@ -696,9 +740,16 @@ private void loadAllSchema() { @Override public void clear() { // Clear schema info firstly - super.clear(); - this.clearCache(false); - this.publishClearEvent(); + try { + super.clear(); + } finally { + // A clear that threw may have removed part of the schema + try { + this.clearCache(false); + } finally { + this.publishClearEvent(); + } + } } /** @@ -729,6 +780,8 @@ static final class PendingClearEvents { private ScheduledExecutorService executor; private ScheduledFuture retry; private long retryDelay = RETRY_MIN_DELAY; + // The number of the newest scheduled retry + private long retryEpoch; void publish(String graphSpace, String graph) { try { @@ -773,11 +826,12 @@ private synchronized void scheduleRetry(boolean now) { }); } long delay = now ? 0L : this.retryDelay; - this.retry = this.executor.schedule(this::retryPending, delay, - TimeUnit.MILLISECONDS); + long epoch = ++this.retryEpoch; + this.retry = this.executor.schedule(() -> this.retryPending(epoch), + delay, TimeUnit.MILLISECONDS); } - private void retryPending() { + private void retryPending(long epoch) { List> entries = new ArrayList<>(this.pending.entrySet()); for (Map.Entry entry : entries) { @@ -794,6 +848,13 @@ private void retryPending() { } } synchronized (this) { + if (epoch != this.retryEpoch) { + /* + * A watch start scheduled a newer retry while this one + * was running: that one goes on, or there would be two. + */ + return; + } this.retry = null; if (this.pending.isEmpty()) { this.retryDelay = RETRY_MIN_DELAY; @@ -816,9 +877,16 @@ private record GraphRef(String graphSpace, String graph) { * Schema caches never expire, so a value cached after it was replaced in * the store stays wrong until the next clear. Every clear, removal and * local write therefore advances {@link #generation} under the write - * lock. A reader notes the generation before it asks the store and caches - * what it got only if the generation is still the same, checked under the - * same lock. The store is never called with a lock held. + * lock. A reader notes the generation before it looks into a cache or + * asks the store, and caches what it got only if the generation is still + * the same, checked under the same lock. The store is never called with + * that lock held. + *

+ * The generation is advanced as the last step of a change, after the + * caches were updated. Readers note it without a lock, so one that sees + * the new generation must also see the caches as the change left them. + * Advancing first would let a reader note the new generation, take the + * old value from the id cache and copy it to the array cache afterwards. */ private static final class SchemaCaches { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java index 53f10f54bb..b88da42beb 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java @@ -77,6 +77,8 @@ public class CachedSchemaTransactionTest extends BaseUnitTest { @Before public void setup() { + // A V2 graph opened without a PD leaves a clear event to send again + CachedSchemaTransactionV2Test.resetPendingClearEvents(); this.graph = HugeFactory.open(FakeObjects.newConfig()); this.params = Whitebox.getInternalState(this.graph, "params"); this.cache = new CachedSchemaTransaction(this.params, @@ -96,6 +98,7 @@ public void teardown() throws Exception { this.graph.close(); this.graph = null; } + CachedSchemaTransactionV2Test.resetPendingClearEvents(); } } @@ -1110,6 +1113,8 @@ public void testV2WarmRequestsRetainSchemaWithoutMetadataReadsAndReceiveInvalida graphManagerReplaced = true; CachedSchemaTransactionV2 first = new CachedSchemaTransactionV2(driver, "test", requestParams); requests.add(first); + // The clear event this JVM sends when it opens the graph + Mockito.verify(driver, Mockito.times(1)).put(Mockito.anyString(), Mockito.anyString()); VertexLabel retainedLabel = first.getVertexLabel(original.id()); Assert.assertEquals(1, reads.get()); HugeVertex retained = new HugeVertex(objects.graph(), IdGenerator.of(32), retainedLabel); @@ -1161,7 +1166,7 @@ public void testV2WarmRequestsRetainSchemaWithoutMetadataReadsAndReceiveInvalida storeHub.listen(EventHub.ANY_EVENT, listener); } } - // One clear event when this JVM opened the graph, one for the Store clear + // One more clear event, for the Store clear Mockito.verify(driver, Mockito.times(2)).put(Mockito.anyString(), Mockito.anyString()); CachedSchemaTransactionV2 afterStoreEvent = new CachedSchemaTransactionV2(driver, "test", requestParams); requests.add(afterStoreEvent); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java index 4f71b35e07..4394628adf 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionV2Test.java @@ -20,6 +20,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -29,12 +30,16 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Lock; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; import java.util.function.BiConsumer; import java.util.function.BiPredicate; +import java.util.function.Consumer; import org.apache.hugegraph.HugeGraph; import org.apache.hugegraph.HugeGraphParams; @@ -51,6 +56,8 @@ import org.apache.hugegraph.id.IdGenerator; import org.apache.hugegraph.meta.PdMetaDriver; import org.apache.hugegraph.meta.managers.SchemaMetaManager; +import org.apache.hugegraph.pd.client.KvClient; +import org.apache.hugegraph.pd.grpc.kv.WatchResponse; import org.apache.hugegraph.struct.schema.PropertyKey; import org.apache.hugegraph.struct.schema.SchemaElement; import org.apache.hugegraph.testutil.Assert; @@ -127,7 +134,7 @@ public void setup() throws Exception { // There is no PD: don't register the meta watch, and record publishes this.originalRegistered = registeredFlag().getAndSet(true); // Other test classes open V2 graphs without a PD and leave failed publishes - pendingMap().clear(); + resetPendingClearEvents(); this.publisher = new RecordingPublisher(); this.originalPublisher = Whitebox.getInternalState( CachedSchemaTransactionV2.class, "clearEventPublisher"); @@ -140,17 +147,21 @@ public void setup() throws Exception { @After public void teardown() throws Exception { - this.store.open(); - this.executor.shutdownNow(); - Assert.assertTrue(this.executor.awaitTermination(WAIT_SECONDS, - TimeUnit.SECONDS)); - this.tx.close(); - CachedSchemaTransactionV2.closeGraph(this.params); - pendingMap().clear(); - Whitebox.setInternalState(CachedSchemaTransactionV2.class, - "clearEventPublisher", this.originalPublisher); - registeredFlag().set(this.originalRegistered); - LockUtil.destroy(this.spaceGraph); + try { + this.store.open(); + this.executor.shutdownNow(); + Assert.assertTrue(this.executor.awaitTermination(WAIT_SECONDS, + TimeUnit.SECONDS)); + this.tx.close(); + CachedSchemaTransactionV2.closeGraph(this.params); + } finally { + // Static state of the class under test: always put it back + resetPendingClearEvents(); + Whitebox.setInternalState(CachedSchemaTransactionV2.class, + "clearEventPublisher", this.originalPublisher); + registeredFlag().set(this.originalRegistered); + LockUtil.destroy(this.spaceGraph); + } } // ------------------------------------------------------------ publish @@ -255,6 +266,105 @@ public void testSchemaClearPublishes() { Assert.assertNull(this.tx.getPropertyKey(IdGenerator.of(1))); } + @Test + public void testEveryPublishComesAfterItsStoreWrite() { + // What another Server would read from the store when the event arrives + List seen = new ArrayList<>(); + this.publisher.intercept = () -> { + PropertyKey stored = this.otherServer.getPropertyKey(GRAPH_SPACE, this.graphName, + IdGenerator.of(1)); + seen.add(stored == null ? -1 : this.version(stored)); + }; + PropertyKey pk = this.propertyKey(1, "name", 1); + this.tx.addPropertyKey(pk); + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + this.tx.removeSchema(pk); + this.tx.addPropertyKey(this.propertyKey(1, "name", 3)); + this.tx.clear(); + this.publisher.intercept = null; + + // added, updated, removed, added again, cleared + Assert.assertEquals(Arrays.asList(1, 2, -1, 3, -1), seen); + } + + // ------------------------------------------------------------ failed write + + @Test + public void testWriteThatFailsHalfwayPublishesAndDropsTheElement() { + Id id = IdGenerator.of(1); + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + this.tx.getPropertyKeys(); + this.publisher.reset(); + + // The id key is replaced, then the store fails before the name key + this.store.hook = (operation, key) -> { + if (operation.equals("put") && key.contains("/ID/1")) { + throw new IllegalStateException("PD is unreachable"); + } + }; + long generation = this.generation(); + Assert.assertThrows(IllegalStateException.class, () -> { + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + }); + this.store.hook = null; + + // The store changed, so the other Servers are told although the call failed + Assert.assertEquals(1, this.publisher.count(this.graphName)); + // A read in flight can't cache what it fetched before the failed write + Assert.assertTrue(this.generation() > generation); + // And this Server asks the store instead of serving the copy it had + Assert.assertNull(this.arrayCached(id)); + long reads = this.store.reads(); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertTrue(this.store.reads() > reads); + } + + @Test + public void testFailedWriteKeepsItsEventWhenThePublishFailsToo() throws Exception { + this.tx.addPropertyKey(this.propertyKey(1, "name", 1)); + this.publisher.reset(); + this.publisher.fail = true; + this.store.hook = (operation, key) -> { + if (operation.equals("put") && key.contains("/ID/1")) { + throw new IllegalStateException("PD is unreachable"); + } + }; + Assert.assertThrows(IllegalStateException.class, () -> { + this.tx.updatePropertyKey(this.propertyKey(1, "name", 2)); + }); + this.store.hook = null; + Assert.assertTrue(this.pending()); + + this.publisher.fail = false; + this.await(() -> this.publisher.count(this.graphName) == 1); + this.await(() -> !this.pending()); + } + + @Test + public void testRemoveThatFailsHalfwayPublishesAndDropsTheElement() { + PropertyKey pk = this.propertyKey(1, "name", 1); + this.tx.addPropertyKey(pk); + Assert.assertNotNull(this.tx.getPropertyKey("name")); + this.publisher.reset(); + + // The name key is deleted, then the store fails before the id key + this.store.hook = (operation, key) -> { + if (operation.equals("delete") && key.contains("/NAME/name")) { + throw new IllegalStateException("PD is unreachable"); + } + }; + Assert.assertThrows(IllegalStateException.class, () -> { + this.tx.removeSchema(pk); + }); + this.store.hook = null; + + Assert.assertEquals(1, this.publisher.count(this.graphName)); + // Not served by name from the cache after the store lost the name + long reads = this.store.reads(); + Assert.assertNull(this.tx.getPropertyKey("name")); + Assert.assertTrue(this.store.reads() > reads); + } + // ------------------------------------------------------------ failed publish @Test @@ -333,8 +443,66 @@ public void testFailureDuringARetryIsKept() throws Exception { Assert.assertEquals(2, this.publisher.count(this.graphName)); } + @Test + public void testWatchStartDuringARetryLeavesOneRetry() throws Exception { + long epoch = pendingState("retryEpoch"); + this.publisher.reset(); + this.publisher.fail = true; + this.tx.updatePropertyKey(this.propertyKey(1, "name", 1)); + + // The first retry is inside its publish when the watch starts again + CountDownLatch retryInPublish = new CountDownLatch(1); + CountDownLatch finishRetry = new CountDownLatch(1); + Thread testThread = Thread.currentThread(); + this.publisher.intercept = () -> { + if (Thread.currentThread() != testThread && retryInPublish.getCount() > 0L) { + retryInPublish.countDown(); + awaitLatch(finishRetry); + } + }; + awaitLatch(retryInPublish); + watchStarted(); + finishRetry.countDown(); + // That retry fails, and so does the one the watch start asked for + this.await(() -> this.publisher.attempts.get() >= 3); + Thread.sleep(300L); + + /* + * Three retries were scheduled: the first, the one of the watch start + * and one follow-up. The retry that was overtaken must not schedule a + * follow-up of its own, or two would run from here on and the delay + * would double twice per round. + */ + Assert.assertEquals(epoch + 3L, pendingState("retryEpoch")); + Assert.assertEquals(2000L, pendingState("retryDelay")); + Assert.assertEquals(3, this.publisher.attempts.get()); + + // The delay starts over once the event got through + this.publisher.intercept = null; + this.publisher.fail = false; + watchStarted(); + this.await(() -> !this.pending()); + this.await(() -> pendingState("retryDelay") == 1000L); + } + // ------------------------------------------------------------ watch start + @Test + public void testPdMetaDriverPassesTheWatchStartedCallback() throws Exception { + PdMetaDriver driver = Mockito.mock(PdMetaDriver.class, Mockito.CALLS_REAL_METHODS); + @SuppressWarnings("unchecked") + KvClient client = Mockito.mock(KvClient.class); + Whitebox.setInternalState(driver, "client", client); + Consumer consumer = response -> { }; + Runnable started = () -> { }; + + driver.listen("key", consumer, started); + + // Without it a Server would never learn that its watch started again + Mockito.verify(client).listen(Mockito.eq("key"), Mockito.same(consumer), + Mockito.any(), Mockito.same(started)); + } + @Test public void testWatchStartClearsEveryGraph() throws Exception { this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, @@ -468,6 +636,101 @@ public void testReadStartedBeforeARemoveIsNotCached() throws Exception { Assert.assertEquals(0, this.tx.getPropertyKeys().size()); } + @Test + public void testReadStartedBeforeALocalInvalidEventIsNotCached() throws Exception { + Id id = IdGenerator.of(1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + + Gate gate = this.store.stopNext("get", "/ID/1"); + Future read = this.executor.submit(() -> this.tx.getPropertyKey(id)); + gate.awaitReached(); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 2)); + // The element is invalidated through the event hub of the graph + this.schemaEventHub.notify(Events.CACHE, Cache.ACTION_INVALID, + HugeType.PROPERTY_KEY, id).get(); + gate.open(); + Assert.assertEquals(1, this.version(read.get(WAIT_SECONDS, TimeUnit.SECONDS))); + + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + } + + @Test + public void testReadDuringALocalWriteDoesNotPutTheOldValueBack() throws Exception { + Id id = IdGenerator.of(1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + // Version 1 is in the id cache and not in the array cache + this.tx.getPropertyKeys(); + Assert.assertNull(this.arrayCached(id)); + + /* + * Hold the write of version 2 while it updates the caches: the store + * has version 2 and the id cache still has version 1. + */ + CountDownLatch inCaches = new CountDownLatch(1); + CountDownLatch finishWrite = new CountDownLatch(1); + AtomicBoolean armed = new AtomicBoolean(false); + PropertyKey update = Mockito.spy(this.propertyKey(1, "name", 2)); + Mockito.doAnswer(invocation -> { + if (armed.compareAndSet(true, false)) { + inCaches.countDown(); + awaitLatch(finishWrite); + } + return HugeType.PROPERTY_KEY; + }).when(update).type(); + this.store.hook = (operation, key) -> { + if (operation.equals("put") && key.contains("/NAME/name")) { + armed.set(true); + } + }; + Future write = this.executor.submit(() -> this.tx.updatePropertyKey(update)); + awaitLatch(inCaches); + + // A read takes version 1 from the id cache and wants to copy it + AtomicReference reader = new AtomicReference<>(); + Future read = this.executor.submit(() -> { + reader.set(Thread.currentThread()); + return this.tx.getPropertyKey(id); + }); + this.await(() -> reader.get() != null && + reader.get().getState() == Thread.State.WAITING); + finishWrite.countDown(); + write.get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.assertEquals(1, this.version(read.get(WAIT_SECONDS, TimeUnit.SECONDS))); + + // It did not: every later read gets version 2 without asking the store + long reads = this.store.reads(); + Assert.assertEquals(2, this.version((PropertyKey) this.arrayCached(id))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + Assert.assertEquals(reads, this.store.reads()); + } + + @Test + public void testGuardIsSharedByTheTransactionsOfAGraph() throws Exception { + Id id = IdGenerator.of(1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + // Each request has its own transaction; they share the caches + CachedSchemaTransactionV2 second = new CachedSchemaTransactionV2( + this.store.driver, CLUSTER, this.params); + try { + Gate gate = this.store.stopNext("get", "/ID/1"); + Future read = this.executor.submit(() -> this.tx.getPropertyKey(id)); + gate.awaitReached(); + second.updatePropertyKey(this.propertyKey(1, "name", 2)); + gate.open(); + Assert.assertEquals(1, this.version(read.get(WAIT_SECONDS, TimeUnit.SECONDS))); + + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(2, this.version(second.getPropertyKey("name"))); + } finally { + second.close(); + } + } + // ------------------------------------------------------------ guard: write-through @Test @@ -512,6 +775,36 @@ public void testAddWriteThroughAfterAClearIsNotCached() throws Exception { Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); } + @Test + public void testRefusedWriteThroughEvictsWhatWasCachedMeanwhile() throws Exception { + Id id = IdGenerator.of(1); + this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, + this.propertyKey(1, "name", 1)); + + // The write of version 2 noted the generation and has not reached the store + Gate gate = this.store.stopBeforePut("/ID/1"); + Future write = this.executor.submit( + () -> this.tx.updatePropertyKey(this.propertyKey(1, "name", 2))); + gate.awaitReached(); + // A clear arrives, and readers cache version 1 before the write lands + this.clearByEvent(); + Assert.assertEquals(1, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(1, this.version(this.tx.getPropertyKeys().get(0))); + gate.open(); + write.get(WAIT_SECONDS, TimeUnit.SECONDS); + + // The write-through is not cached, and it removed the older copy + Assert.assertNull(this.arrayCached(id)); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey(id))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKey("name"))); + Assert.assertEquals(2, this.version(this.tx.getPropertyKeys().get(0))); + + // That listing is the whole list again: the next one needs no store + long scans = this.store.scans.get(); + Assert.assertEquals(1, this.tx.getPropertyKeys().size()); + Assert.assertEquals("served from the cache", scans, this.store.scans.get()); + } + @Test public void testWriteThroughWithoutAClearIsCached() { Id id = IdGenerator.of(1); @@ -654,27 +947,42 @@ public void testAClearWaitsForAListingReadFromTheCache() throws Exception { this.propertyKey(1, "name", 1)); this.tx.getPropertyKeys(); - // Stand in for a listing that is reading the cached list - Lock readLock = (Lock) this.callGuard("readLock"); - readLock.lock(); - Future clear; - try { - clear = this.executor.submit(this::clearByEvent); - try { - clear.get(300L, TimeUnit.MILLISECONDS); - Assert.fail("the clear ran while a listing was reading the cache"); - } catch (java.util.concurrent.TimeoutException expected) { - // The clear waits + // A cached element that can hold a listing in the middle of its pass + CountDownLatch inListing = new CountDownLatch(1); + CountDownLatch finishListing = new CountDownLatch(1); + AtomicBoolean armed = new AtomicBoolean(false); + PropertyKey slow = Mockito.spy(this.propertyKey(2, "slow", 1)); + Mockito.doAnswer(invocation -> { + if (armed.get()) { + inListing.countDown(); + awaitLatch(finishListing); } + return HugeType.PROPERTY_KEY; + }).when(slow).type(); + this.invoke("cacheIfCurrent", new Class[]{SchemaElement.class, long.class}, + slow, this.generation()); + armed.set(true); + + Future> listing = this.executor.submit( + () -> this.tx.getPropertyKeys()); + awaitLatch(inListing); + Future clear = this.executor.submit(this::clearByEvent); + try { + clear.get(300L, TimeUnit.MILLISECONDS); + Assert.fail("the clear ran while a listing was reading the cache"); + } catch (TimeoutException expected) { + // The clear waits for the listing } finally { - readLock.unlock(); + finishListing.countDown(); } + Assert.assertEquals(2, listing.get(WAIT_SECONDS, TimeUnit.SECONDS).size()); clear.get(WAIT_SECONDS, TimeUnit.SECONDS); } @Test public void testListingIsNeverPartialWhileClearsRun() throws Exception { int count = 50; + int listings = 3000; for (int i = 1; i <= count; i++) { this.otherServer.addPropertyKey(GRAPH_SPACE, this.graphName, this.propertyKey(i, "pk" + i, 1)); @@ -683,42 +991,58 @@ public void testListingIsNeverPartialWhileClearsRun() throws Exception { Future clears = this.executor.submit(() -> { while (!stop.get()) { this.clearByEvent(); + // Leave room for a listing to fill the cache and for others to use it + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1L)); } }); + long scans = this.store.scans.get(); try { - for (int i = 0; i < 3000; i++) { + for (int i = 0; i < listings; i++) { Assert.assertEquals("listing " + i, count, this.tx.getPropertyKeys().size()); } } finally { stop.set(true); } clears.get(WAIT_SECONDS, TimeUnit.SECONDS); + Assert.assertTrue("some listings must have come from the cache", + this.store.scans.get() - scans < listings); } @Test - public void testReadsAndWritesStayConsistentUnderConcurrentClears() throws Exception { + public void testReadsNeverGoBackBehindAWriteUnderConcurrentClears() throws Exception { Id id = IdGenerator.of(1); this.tx.addPropertyKey(this.propertyKey(1, "name", 0)); + // The last version whose update has returned + AtomicInteger written = new AtomicInteger(0); AtomicBoolean stop = new AtomicBoolean(false); List> noise = new ArrayList<>(); noise.add(this.executor.submit(() -> { while (!stop.get()) { this.clearByEvent(); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1L)); } })); for (int i = 0; i < 3; i++) { noise.add(this.executor.submit(() -> { while (!stop.get()) { - this.tx.getPropertyKey(id); - this.tx.getPropertyKey("name"); - this.tx.getPropertyKeys(); + int floor = written.get(); + this.assertNotOlder(floor, this.tx.getPropertyKey(id)); + floor = written.get(); + this.assertNotOlder(floor, this.tx.getPropertyKey("name")); + floor = written.get(); + this.assertNotOlder(floor, this.tx.getPropertyKeys().get(0)); } })); } - try { + Future writer = this.executor.submit(() -> { for (int version = 1; version <= 300; version++) { this.tx.updatePropertyKey(this.propertyKey(1, "name", version)); + written.set(version); } + }); + try { + // A deadlock fails here instead of hanging the run + writer.get(3L * WAIT_SECONDS, TimeUnit.SECONDS); } finally { stop.set(true); } @@ -726,7 +1050,6 @@ public void testReadsAndWritesStayConsistentUnderConcurrentClears() throws Excep future.get(WAIT_SECONDS, TimeUnit.SECONDS); } - // Whatever ended up cached, it is the last version written Assert.assertEquals(300, this.version(this.tx.getPropertyKey(id))); Assert.assertEquals(300, this.version(this.tx.getPropertyKey("name"))); Assert.assertEquals(300, this.version(this.tx.getPropertyKeys().get(0))); @@ -745,6 +1068,15 @@ private int version(PropertyKey pk) { return ((Number) pk.userdata().get(VERSION)).intValue(); } + // A read that began after a version was written must not return an older one + private void assertNotOlder(int floor, PropertyKey pk) { + int version = this.version(pk); + if (version < floor) { + throw new AssertionError("read version " + version + " after version " + + floor + " was written"); + } + } + /** What this Server does when another Server's clear event arrives. */ private void clearByEvent() { Whitebox.invokeStatic(CachedSchemaTransactionV2.class, @@ -810,10 +1142,38 @@ private static AtomicBoolean registeredFlag() throws Exception { return (AtomicBoolean) field.get(null); } + private static Object pendingClearEvents() { + return Whitebox.getInternalState(CachedSchemaTransactionV2.class, + "PENDING_CLEAR_EVENTS"); + } + private static Map pendingMap() { - Object pending = Whitebox.getInternalState(CachedSchemaTransactionV2.class, - "PENDING_CLEAR_EVENTS"); - return Whitebox.getInternalState(pending, "pending"); + return Whitebox.getInternalState(pendingClearEvents(), "pending"); + } + + private static long pendingState(String field) { + Object pending = pendingClearEvents(); + synchronized (pending) { + Long value = Whitebox.getInternalState(pending, field); + return value; + } + } + + /** + * Forget the failed publishes and their scheduled retry. They are static + * state of the class under test, so without this a test that opens a V2 + * graph without a PD leaves a retry behind that fires in a later test. + */ + static void resetPendingClearEvents() { + Object pending = pendingClearEvents(); + synchronized (pending) { + pendingMap().clear(); + ScheduledFuture retry = Whitebox.getInternalState(pending, "retry"); + if (retry != null) { + retry.cancel(false); + } + Whitebox.setInternalState(pending, "retryDelay", 1000L); + } } // Whether this graph has a clear event waiting to be published again @@ -919,10 +1279,15 @@ private static final class FakeStore { private final List gates = new CopyOnWriteArrayList<>(); private final PdMetaDriver driver = Mockito.mock(PdMetaDriver.class); private volatile StoreHook hook; + private volatile Consumer beforePut; FakeStore() { Mockito.doAnswer(invocation -> { String key = invocation.getArgument(0); + Consumer before = this.beforePut; + if (before != null) { + before.accept(key); + } this.data.put(key, invocation.getArgument(1)); this.after("put", key); return null; @@ -979,8 +1344,22 @@ Gate stopNext(String operation, String keyPart) { return gate; } + /** Stop the next put whose key contains the text, before it changes the store. */ + Gate stopBeforePut(String keyPart) { + Gate gate = new Gate(); + AtomicBoolean used = new AtomicBoolean(false); + this.gates.add(gate); + this.beforePut = key -> { + if (key.contains(keyPart) && used.compareAndSet(false, true)) { + gate.stop(); + } + }; + return gate; + } + void open() { this.hook = null; + this.beforePut = null; for (Gate gate : this.gates) { gate.open(); } From 7df011acba97dd5020e9e903f6acc7c8333d45c6 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Fri, 13 Nov 2026 18:42:20 +0530 Subject: [PATCH 3/3] docs(server): list the stopped-watch limit and tighten the index test Follow-up to review comments on this change. - MultiClusterSchemaCacheTest: the readers that run while an index is created were submitted and never checked, so a reader that stopped on an error left the test passing with fewer reads. Their futures are read once the index is created. - docs/server-runtime.md: a PD watch that ends with a gRPC status the PD client does not retry is not started again, so that Server keeps its cached schema until it is restarted. This is how master behaves for every meta listener and is tracked in #3478. - A TODO in the test's waitFor notes that the shared RestClient sets no timeouts. --- docs/server-runtime.md | 5 +++++ .../MultiClusterSchemaCacheTest.java | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/server-runtime.md b/docs/server-runtime.md index 5cb0da6f16..2682db9f84 100644 --- a/docs/server-runtime.md +++ b/docs/server-runtime.md @@ -873,6 +873,11 @@ What it does not provide: - A Server that cannot reach PD keeps serving its cached schema until it reconnects. - A PD watch that dies without an error is noticed only when another PD call from that Server fails. +- A PD watch that ends with an error the PD client does not retry, such as + `UNAUTHENTICATED`, `PERMISSION_DENIED` or `UNIMPLEMENTED`, is not started again. The + Server logs the stopped watch at ERROR level and keeps its cached schema until it is + restarted. PD does not answer a watch with these codes itself; they come from the + gRPC layer, for example from a proxy between Server and PD. - With the etcd meta driver the watch does not report restarts. A Server then does not drop its caches after a watch restart, and a failed publish is sent again by the timed retry only. diff --git a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java index 1dcc478210..83cf685dca 100644 --- a/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java +++ b/hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java @@ -22,6 +22,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -146,16 +147,17 @@ public void testIndexCreatedThroughOneServerWorksOnTheOthers() throws Exception // The other Servers serve reads of this label while the index is created AtomicBoolean stop = new AtomicBoolean(false); ExecutorService readers = Executors.newFixedThreadPool(4); + List> reads = new ArrayList<>(); for (int reader = 0; reader < 4; reader++) { RestClient client = clients.get(1 + reader % 2); - readers.submit(() -> { + reads.add(readers.submit(() -> { while (!stop.get()) { Response r = client.target().path(VERTICES) .queryParam("label", vl) .queryParam("limit", 3).request().get(); r.readEntity(String.class); } - }); + })); } try { JsonNode created = json(clients.get(0).post(ILS, @@ -171,6 +173,10 @@ public void testIndexCreatedThroughOneServerWorksOnTheOthers() throws Exception readers.shutdown(); Assert.assertTrue(readers.awaitTermination(60L, TimeUnit.SECONDS)); } + // A reader that stopped on an error left the index created with fewer reads + for (Future read : reads) { + read.get(); + } waitFor(il + " created and listed in its label on every Server", VISIBLE_SECONDS, () -> { @@ -386,6 +392,8 @@ private static void waitFor(String what, long seconds, Callable conditi Throwable last = null; while (System.nanoTime() < deadline) { try { + // TODO: RestClient sets no timeouts, so a request that gets no + // answer outlasts this deadline until the suite timeout ends it if (condition.call()) { return; }