diff --git a/CHANGELOG.md b/CHANGELOG.md index de4ff77c..b731bf49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ terminates with an error. Providing not positive timeout value throws an exception. - Fixed connection close callbacks executing before the connection state becomes closed. - Fixed greeting object remaining non-null after connection closure. +- Fixed the `VshardClusterContainer` readiness check: it read a `replicas` field that + `vshard.router.info()` does not return, so the container could be reported ready while a + replicaset master was still in the vshard handshake, and the first CRUD request failed with + `VHANDSHAKE_NOT_COMPLETE`. The check now makes a synchronous `callrw` to every replicaset master, + which reopens a router connection that vshard closed after a failed handshake and waits for the + new one, requires every master to be `available`, and logs the replica and bucket states. ### Documentation diff --git a/tarantool-client/src/test/java/io/tarantool/client/integration/TarantoolCrudClientTest.java b/tarantool-client/src/test/java/io/tarantool/client/integration/TarantoolCrudClientTest.java index 2763380e..8579ef75 100644 --- a/tarantool-client/src/test/java/io/tarantool/client/integration/TarantoolCrudClientTest.java +++ b/tarantool-client/src/test/java/io/tarantool/client/integration/TarantoolCrudClientTest.java @@ -2538,10 +2538,11 @@ public void testSelectWrongParameters() { CrudException cause = (CrudException) ex.getCause(); assertEquals(CompletionException.class, ex.getClass()); assertEquals(CrudException.class, cause.getClass()); + // checks >= 287c88e omits the " to nil" suffix when the function name is unknown. assertTrue( cause .getMessage() - .matches("SelectError:.*" + "unexpected argument opts.coolOption to nil")); + .matches("SelectError:.*unexpected argument opts\\.coolOption( to nil)?")); } @SuppressWarnings("unchecked") diff --git a/testcontainers/src/main/java/org/testcontainers/containers/cluster/vshard/VshardClusterContainer.java b/testcontainers/src/main/java/org/testcontainers/containers/cluster/vshard/VshardClusterContainer.java index 481580d8..480b53a9 100644 --- a/testcontainers/src/main/java/org/testcontainers/containers/cluster/vshard/VshardClusterContainer.java +++ b/testcontainers/src/main/java/org/testcontainers/containers/cluster/vshard/VshardClusterContainer.java @@ -65,16 +65,39 @@ public class VshardClusterContainer extends GenericContainer 0 then return false end;" - + " return true"; + "local vshard = require('vshard')" + + " local replicasets = vshard.router.routeall()" + + " local ready = next(replicasets) ~= nil" + + " local parts = {}" + + " for id, rs in pairs(replicasets) do" + + " local res, err = rs:callrw('vshard.storage.buckets_count', {}, {timeout = 2})" + + " if res == nil then" + + " ready = false" + + " table.insert(parts, tostring(id) .. ' callrw failed: ' .. tostring(err))" + + " end" + + " end" + + " local info = vshard.router.info()" + + " for id, rs in pairs(info.replicasets or {}) do" + + " local m = rs.master and rs.master.status or 'nil'" + + " local r = rs.replica and rs.replica.status or 'nil'" + + " if m ~= 'available' then ready = false end" + + " table.insert(parts, tostring(id) .. ' master=' .. m .. ' replica=' .. r)" + + " end" + + " local b = info.bucket or {}" + + " table.insert(parts, 'buckets rw=' .. tostring(b.available_rw)" + + " .. ' ro=' .. tostring(b.available_ro)" + + " .. ' unreachable=' .. tostring(b.unreachable)" + + " .. ' unknown=' .. tostring(b.unknown))" + + " return ready, table.concat(parts, '; ')"; private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory()); public static final String ENV_TARANTOOL_VERSION = "TARANTOOL_VERSION"; @@ -569,13 +592,14 @@ protected void waitUntilVshardIsBootstrapped(int secondsToWait) { } /** - * Waits until every vshard replica is in the {@code available} state (i.e. the vshard handshake - * with each storage has completed) and there are no unreachable buckets. {@link - * #vshardIsBootstrapped()} only verifies that {@code vshard.router.bootstrap} returned cleanly, - * which does not preclude individual storages from still being in {@code VHANDSHAKE_NOT_COMPLETE} - * (vshard code 40) during the initial rebalance. Issuing a CRUD request against such a cluster - * fails immediately, so the readiness check must additionally inspect the per-replica status - * surfaced by {@code vshard.router.info()}. + * Waits until a synchronous {@code callrw} to the master of every replicaset succeeds and every + * master is {@code available} in {@code vshard.router.info()}. {@link #vshardIsBootstrapped()} + * only verifies that {@code vshard.router.bootstrap} returned cleanly. The router may still hold + * a closed connection to a master: vshard closes it when the handshake fails (for example, the + * storage was not configured yet when the router connected), and only a request to that master + * reopens it. An asynchronous CRUD map call reopens it but does not wait for the new handshake + * and fails with {@code VHANDSHAKE_NOT_COMPLETE} (vshard code 40), so the probe makes the + * synchronous call itself. */ protected void waitUntilVshardStoragesAreReady(int secondsToWait) { if (!waitUntilTrue(secondsToWait, this::vshardStoragesAreReady)) { @@ -658,9 +682,10 @@ protected boolean vshardIsBootstrapped() { } /** - * Returns {@code true} when {@code vshard.router.info()} reports every replica as {@code - * available} and {@code info.bucket.unreachable == 0}, i.e. the vshard handshake has completed - * for all storages. See {@link #waitUntilVshardStoragesAreReady(int)} for rationale. + * Returns {@code true} when a synchronous {@code callrw} to every replicaset master succeeds and + * {@code vshard.router.info()} reports every master as {@code available}. Call errors, replica + * and bucket states returned by the probe are only logged. See {@link + * #waitUntilVshardStoragesAreReady(int)} for rationale. */ protected boolean vshardStoragesAreReady() { try { @@ -672,8 +697,11 @@ protected boolean vshardStoragesAreReady() { return false; } boolean ready = Boolean.TRUE.equals(result.get(0)); - if (!ready) { - logger().warn("Vshard storages are not handshaked yet"); + Object state = result.get(1); + if (ready) { + logger().info("Vshard storages are ready: {}", state); + } else { + logger().warn("Vshard storages are not handshaked yet: {}", state); } return ready; } catch (Exception e) {