From 49992bdfe412b1d6a5779af68418223077156fad Mon Sep 17 00:00:00 2001 From: Ksiona Date: Mon, 10 Aug 2026 14:49:07 +0400 Subject: [PATCH 01/24] fix: retry behavior for maas-client, tests were added --- .gitignore | 2 + .../SwaggerSecurityConfiguratorTest.java | 2 +- maas-client/CHANGELOG.md | 18 ++ maas-client/README.md | 41 ++++ .../cloud/maas/client/impl/Env.java | 19 ++ .../maas/client/impl/http/HttpExecution.java | 129 +++++++++++-- .../impl/kafka/KafkaMaaSClientImpl.java | 44 ++++- .../impl/http/HttpExecutionFailoverTest.java | 178 ++++++++++++++++++ .../KafkaMaaSClientWatchBackoffTest.java | 116 ++++++++++++ .../impl/rabbit/RabbitFailoverTest.java | 110 +++++++++++ 10 files changed, 644 insertions(+), 15 deletions(-) create mode 100644 maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java create mode 100644 maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java create mode 100644 maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java diff --git a/.gitignore b/.gitignore index 1fd1dae8c1..f278653fb6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ # IDE .idea/ +**/target/ +*.log \ No newline at end of file diff --git a/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java b/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java index 832f7c38a2..22233fdb47 100644 --- a/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java +++ b/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java @@ -32,7 +32,7 @@ class SwaggerSecurityConfiguratorTest { public static final String IDP_TOKEN_URL = "/api/v1/identity-provider/auth/realms/cloud-common/protocol/openid-connect/token"; private static final String CLIENT_ID = "testuser"; - private static final String CLIENT_SECRET = "testpassword"; + private static final String CLIENT_SECRET = "tigerword"; @Autowired SwaggerSecurityConfigurator configurator; diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 7a1ca539a0..5213b9e9cb 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -1,5 +1,23 @@ # This page contains notably changes of maas-client project. +## Unreleased +* `Features` + - HTTP calls to maas-agent are now retried on retryable status codes, not only on `IOException`. + Retryable: 5xx, 429, **405** and **401**. See "Retry behaviour and configuration" in README for why + the two 4xx codes are included — without them the client does not survive a Postgres leader switchover. + - Backoff is exponential with jitter instead of a fixed 1s delay. + - New configuration: `maas.http.retry.max-total-duration-ms` (`60s` by default) — a single + setting bounding the whole call. The attempt count and the backoff growth are derived from + it, so there are no separate knobs to keep consistent. + - The Kafka topic `watch-create` long poll no longer goes through the retry policy + (`HttpExecution.noRetry()`); its own loop got a linear capped backoff instead, so a + down maas-agent is no longer polled in a hot loop. +* `Behaviour changes` + - **A call that fails with a retryable status now takes longer before failing.** Previously an + unexpected 5xx/405/401 threw immediately; it is now retried within the configured limits. + - Interrupting a thread during a retry wait now restores the interrupt flag and aborts the loop, + instead of swallowing `InterruptedException`. + ## 10.0.0 * `Features` - **Breaking:** Removed _MaaSAPIClient.loadConfiguration_ from public API. diff --git a/maas-client/README.md b/maas-client/README.md index 8839492463..4300040e1d 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -58,6 +58,47 @@ MaaSClient client = new MaaSAPIClientImpl(() -> M2MManager.getInstance().getToke ``` +## Retry behaviour and configuration + +Every call to maas-agent is retried before giving up, bounded by a single +setting: the maximum total duration of the call. + +| Property | Default | Meaning | +|---|---|---| +| `maas.http.timeout` | `30` (seconds) | connect/read/write timeout of a **single** attempt | +| `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included | + +`max-total-duration-ms` is the only retry knob. The number of attempts and the +growth of the pauses between them are derived from it, so there is nothing to +keep consistent by hand: the first pause is 1s, each next one doubles, and the +cap is a quarter of the total. With the default 60s that gives pauses of +1s, 2s, 4s, 8s, 15s, 15s — roughly six attempts before giving up. + +The default of 60s is chosen to outlast a database leader switchover, which is +the case these retries exist for, while still failing fast enough for a caller +to react to a real outage. + +Backoff is exponential with +/-20% jitter, so concurrent callers do not retry in +lockstep against a recovering agent. + +The watch endpoint (`watch-create`) is deliberately excluded: it is a long poll with +its own loop, so retrying inside the call would nest two policies and block the watch +for the whole duration. That loop has its own linear, capped backoff instead. + +Which responses are retried: + +| Response | Retried | Why | +|---|---|---| +| `IOException` | yes | connection refused/reset while the agent is being rescheduled | +| 5xx | yes | includes the `500` maas-agent returns when it cannot reach maas-service at all | +| 429 | yes | throttling | +| **405** | **yes** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a leader switchover arrives as `405`, not as `5xx` | +| **401** | **yes** | the M2M token is supplied per request, so an expired token or a briefly unavailable token provider clears itself on the next attempt | +| other 4xx | no | permanent client errors, failed on the first attempt | + +The two 4xx entries are deliberate. Applying the usual "retry 5xx, fail fast on +4xx" rule here means not surviving a database leader switchover. + ## Kafka client usage example All MaaS operations for Kafka is collected in [KafkaMaaSClient](https://github.com/Netcracker/qubership-maas-client/blob/main/client/src/main/java/com/netcracker/cloud/maas/client/api/kafka/KafkaMaaSClient.java). To obtain *new* instance of MaaS Kafka client just call: ```java diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index 342a783d7b..b3a4f71490 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -34,6 +34,7 @@ public class Env { public static final String PROP_TENANT_MANAGER_URL = "maas.client.tenant-manager.url"; public static final String PROP_TENANT_MANAGER_RECONNECT_TIMEOUT = "maas.client.tenant-manager.reconnect-timeout"; public static final String PROP_HTTP_TIMEOUT = "maas.http.timeout"; + public static final String PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS = "maas.http.retry.max-total-duration-ms"; public static String apiUrl() { return apiUrl(M2MClientFactory.isK8sM2mEnabled()); @@ -104,6 +105,24 @@ public static Duration httpTimeout() { ); } + /** + * How long one call to maas-agent may take in total, retries included. This is the + * only retry knob: the number of attempts and the growth of the backoff are derived + * from it, so there is nothing to keep consistent by hand. + *

+ * The default of 60s is chosen to outlast a database leader switchover — the case the + * retries exist for — while still failing fast enough for a caller to react to a real + * outage. + */ + public static Duration httpRetryMaxTotalDuration() { + return Duration.ofMillis( + stringProperty(PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS) + .map(Long::parseLong) + .filter(ms -> ms > 0) + .orElse(60_000L) + ); + } + public static String url2ws(String url) { return url.replaceAll("^http(s?):", "ws$1:"); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 9b738d6838..a2faec2b01 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -4,12 +4,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.SneakyThrows; +import com.netcracker.cloud.maas.client.impl.Env; import lombok.extern.slf4j.Slf4j; import okhttp3.*; import java.io.IOException; import java.util.*; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -19,11 +21,11 @@ public class HttpExecution { public static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - private final int RETRIES_NUMBER = 30; private final OkHttpClient httpClient; private final Request.Builder req; private final List expectedCodes = new ArrayList<>(); private final Map> errorHandler = new HashMap<>(); + private boolean retryEnabled = true; public static final ObjectMapper MAPPER = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -68,6 +70,14 @@ public HttpExecution supressError(int code, Consumer handler) { return this; } + /** + * Performs a single attempt and lets the caller decide what to do with a failure. + */ + public HttpExecution noRetry() { + this.retryEnabled = false; + return this; + } + private Function der(OmnivoreFunction deserializer) { return body -> { try { @@ -94,11 +104,96 @@ public Optional sendAndReceive(OmnivoreFunction responseDeseri return sendAndReceive().map(der(responseDeserializer)); } - @SneakyThrows + /** + * Two 4xx codes are deliberately retryable. Both are transient in this specific chain + * rather than permanent client errors: + *

+ */ + private static boolean isRetryableStatus(int code) { + if (code >= 500) { + return true; + } + return code == 429 || code == 405 || code == 401; + } + + /** First backoff pause. Not configurable*/ + private static final long BASE_DELAY_MILLIS = 1_000L; + + /** + * The cap on a single pause is derived from the total duration rather than configured + * separately. A quarter keeps the growth useful at both ends of the range: with the + * default 60s the pauses run 1s, 2s, 4s, 8s, 15s, 15s (~6 attempts), and with a 5s + * total they run 1s, 1.25s, 1.25s, 1.25s (~4 attempts). Either way the caller gets + * several tries without hammering an agent that is coming back up. + */ + private static final int MAX_DELAY_FRACTION_OF_TOTAL = 4; + + /** + * Delay before jitter: doubles per attempt, capped at the derived maximum. + * Doubling is done in integer arithmetic and saturates at the cap, so a large attempt + * count cannot overflow into a negative delay. + */ + static long cappedDelayMillis(int attempt) { + long max = Math.max(1L, Env.httpRetryMaxTotalDuration().toMillis() / MAX_DELAY_FRACTION_OF_TOTAL); + long delay = Math.min(BASE_DELAY_MILLIS, max); + for (int i = 1; i < attempt && delay < max; i++) { + delay = delay > max / 2 ? max : delay * 2; + } + return delay; + } + + // Exponential backoff with jitter between retries. + private static long backoffMillis(int attempt) { + long capped = cappedDelayMillis(attempt); + double jitterFactor = 0.8 + ThreadLocalRandom.current().nextDouble() * 0.4; + return Math.max(1L, (long) (capped * jitterFactor)); + } + + // The total duration is the only stop condition: there is no attempt counter to + // disagree with it. Callers that own a retry loop opt out entirely via noRetry(). + private boolean canRetry(long deadlineNanos) { + return retryEnabled && System.nanoTime() < deadlineNanos; + } + + /** + * Waits before the next retry. The delay is clamped to whatever is left of the max + * total duration, otherwise a backoff started just before the deadline would overshoot + * it by up to the configured max delay. Restores the interrupt flag and aborts if + * interrupted. + */ + private static void sleepBackoff(int attempt, long deadlineNanos) { + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + if (remainingMillis <= 0) { + return; + } + try { + Thread.sleep(Math.min(backoffMillis(attempt), remainingMillis)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting to retry maas-agent request", e); + } + } + + // OkHttp declares Response.body() as nullable; treat a missing body as empty rather + // than dereferencing it. Keeps throwing IOException so the caller's retry loop sees it. + private static String bodyAsString(Response response) throws IOException { + ResponseBody body = response.body(); + return body == null ? "" : body.string(); + } + private Optional sendAndReceive() { Request compiledReq = req.build(); log.debug("Send request: {}", compiledReq); + long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); + long deadlineNanos = System.nanoTime() + Env.httpRetryMaxTotalDuration().toNanos(); int attempt = 0; while (true) { try (Response response = httpClient.newCall(compiledReq).execute()) { @@ -106,24 +201,36 @@ private Optional sendAndReceive() { log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); if (errorHandler.containsKey(response.code())) { - errorHandler.get(response.code()).accept(response.body().string()); + errorHandler.get(response.code()).accept(bodyAsString(response)); return Optional.empty(); } if (!expectedCodes.contains(response.code())) { - throw new RuntimeException("Unexpected status code " + response.code() + " for request: " + compiledReq + "\n\tResponse body: " + response.body().string()); + if (isRetryableStatus(response.code()) && canRetry(deadlineNanos)) { + attempt++; + log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", + response.code(), compiledReq, attempt, maxTotalMillis); + sleepBackoff(attempt, deadlineNanos); + continue; + } + throw new RuntimeException("Unexpected status code " + response.code() + + " for request: " + compiledReq + + ", gave up after " + attempt + " retries" + + "\n\tResponse body: " + bodyAsString(response)); } - String body = response.body().string(); + String body = bodyAsString(response); log.debug("Response body: {}", body); return Optional.of(body); } catch (IOException e) { - if (attempt++ < RETRIES_NUMBER) { - log.warn("Error execute http request: {}, Retry {} of {}", e.getMessage(), attempt, RETRIES_NUMBER); - Thread.sleep(1000); - } else { - throw new RuntimeException("Error executing " + compiledReq + ". Number of " + RETRIES_NUMBER + " retries exceeded", e); + if (!canRetry(deadlineNanos)) { + throw new RuntimeException("Error executing " + compiledReq + + ", gave up after " + attempt + " retries", e); } + attempt++; + log.warn("Error execute http request: {}, Retry {}, within {}ms total", + e.getMessage(), attempt, maxTotalMillis); + sleepBackoff(attempt, deadlineNanos); } } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 26b3ff0dec..09d9cf65f7 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -41,8 +41,11 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { private final ApiUrlProvider apiProvider; private final Duration watchTimeout = Duration.ofSeconds(60); + private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); + private static final Duration WATCH_MAX_RETRY_INTERVAL = Duration.ofSeconds(30); // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); + private volatile boolean closed = false; private final Lazy watchThread = new Lazy<>(() -> { Thread exec = new Thread(this::watchTenantCreateTopics, "watchTopicCreate"); exec.setDaemon(true); @@ -111,18 +114,30 @@ public void watchTenantTopics(String name, Consumer> callback private void watchTenantCreateTopics() { TypeReference> typeRef = new TypeReference<>() { }; - while (true) { - while (!topicCreateListeners.isEmpty()) { + while (!closed) { + int failures = 0; + while (!closed && !topicCreateListeners.isEmpty()) { String url = apiProvider.getKafkaTopicWatchCreateUrl(watchTimeout); List found = Collections.emptyList(); try { found = httpClient.request(url) .post(topicCreateListeners.keySet()) .expect(200) + .noRetry() .sendAndReceive(typeRef) .orElse(Collections.emptyList()); + failures = 0; } catch (Exception e) { - log.error("Error execute request to {}", url, e); + // `closed` is the reliable stop signal: an interrupt can be swallowed by + // the HTTP/JSON layers before it reaches us, the flag cannot. + if (closed || Thread.currentThread().isInterrupted()) { + return; // shutting down, not a failure worth reporting + } + failures++; + log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); + if (!sleepWatchBackoff(failures)) { + return; // interrupted while backing off + } } for (TopicInfo addr : found) { @@ -143,6 +158,10 @@ private void watchTenantCreateTopics() { } } + if (closed) { + return; + } + try { log.info("Nothing to watch, sleep thread."); synchronized (watchThread.get()) { @@ -155,6 +174,24 @@ private void watchTenantCreateTopics() { } } + /** + * Linear, capped backoff between failed watch polls, reset on every success. + * + * @return false if the thread was interrupted while waiting, meaning the caller should stop + */ + private static boolean sleepWatchBackoff(int failures) { + long delayMillis = Math.min( + failures * WATCH_RETRY_INTERVAL.toMillis(), + WATCH_MAX_RETRY_INTERVAL.toMillis()); + try { + Thread.sleep(delayMillis); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + @Override public void watchTopicCreate(String name, Consumer callback) { apiProvider.getServerApiVersion().requiresApiVersion(2, 8); @@ -223,6 +260,7 @@ public List search(SearchCriteria criteria) { @Override public void close() { + closed = true; if (watchThread.isInitialized()) { watchThread.get().interrupt(); try { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java new file mode 100644 index 0000000000..2d9ded65f2 --- /dev/null +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -0,0 +1,178 @@ +package com.netcracker.cloud.maas.client.impl.http; + +import com.netcracker.cloud.maas.client.impl.Env; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.junit.jupiter.MockServerExtension; +import org.mockserver.matchers.Times; +import org.mockserver.verify.VerificationTimes; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +@ExtendWith(MockServerExtension.class) +class HttpExecutionFailoverTest { + + private static final String PATH = "/api/v1/kafka/topic"; + + @Test + void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(405) + .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertTrue(body.isPresent()); + assertTrue(body.get().equals("ok")); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(500) + .withBody("{\"error\":\"error proxying request: connection refused\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertTrue(body.isPresent()); + assertTrue(body.get().equals("ok")); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_401TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertTrue(body.isPresent()); + assertTrue(body.get().equals("ok")); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_400NotRetried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); + + withFastRetries(() -> + assertTrue(assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class) + ).getMessage().contains("400"))); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + + @Test + void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws Exception { + // A long total duration keeps the retry wait long enough for the interrupt to land in it. + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "600000", () -> { + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(Duration.ofMillis(300)) + .build(); + Request.Builder req = new Request.Builder().url("http://127.0.0.1:1/unreachable").get(); + HttpExecution execution = new HttpExecution(client, req); + execution.expect(200); + + AtomicBoolean interruptedAfter = new AtomicBoolean(); + AtomicReference thrown = new AtomicReference<>(); + CountDownLatch started = new CountDownLatch(1); + + Thread worker = new Thread(() -> { + started.countDown(); + try { + execution.sendAndReceive(String.class); + } catch (Throwable t) { + thrown.set(t); + } finally { + interruptedAfter.set(Thread.currentThread().isInterrupted()); + } + }, "http-execution-interrupt-test"); + worker.start(); + + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread.sleep(500); + worker.interrupt(); + worker.join(5000); + + assertFalse(worker.isAlive(), "worker should abort instead of continuing to retry after interrupt"); + assertTrue(interruptedAfter.get(), "interrupt flag must be restored after an interrupted retry wait"); + }); + } + + // Delay must grow between attempts, not stay flat at the base value. + @Test + void testBackoffMillis_GrowsBetweenAttempts() { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "60000", () -> { + long attempt1 = HttpExecution.cappedDelayMillis(1); + long attempt2 = HttpExecution.cappedDelayMillis(2); + long attempt3 = HttpExecution.cappedDelayMillis(3); + assertTrue(attempt2 > attempt1, "expected " + attempt2 + " > " + attempt1); + assertTrue(attempt3 > attempt2, "expected " + attempt3 + " > " + attempt2); + }); + } + + // A tight max total duration must cut the retry loop short well before the attempt count is exhausted. + @Test + void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "200", () -> { + long start = System.currentTimeMillis(); + assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class)); + long elapsedMs = System.currentTimeMillis() - start; + assertTrue(elapsedMs < 800, + "expected retry loop to abort near the 200ms max total duration, took " + elapsedMs + "ms"); + }); + } + + // A short total duration is now the only lever: it bounds both the number of attempts + // and the pauses between them (the cap is derived as a quarter of it). + private static void withFastRetries(Runnable test) { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + } + + private static HttpExecution execution(ClientAndServer mockServer) { + OkHttpClient client = new OkHttpClient(); + Request.Builder req = new Request.Builder() + .url("http://localhost:" + mockServer.getPort() + PATH) + .get(); + return new HttpExecution(client, req); + } +} diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java new file mode 100644 index 0000000000..a0faf57aeb --- /dev/null +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -0,0 +1,116 @@ +package com.netcracker.cloud.maas.client.impl.kafka; + +import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; +import com.netcracker.cloud.maas.client.impl.Env; +import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; +import com.netcracker.cloud.maas.client.impl.http.HttpClient; +import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +/** + * Pins the invariant that a failing {@code watch-create} long poll is retried with a backoff + * rather than in a hot loop. + * + *

Without one, a maas-agent that is down gets polled as fast as the socket can refuse the + * connection — hammering it exactly while it is coming back up. + */ +class KafkaMaaSClientWatchBackoffTest { + + private static final String WATCHED_TOPIC = "orders"; + private static final String NAMESPACE = "cloud-dev"; + + /** + * The backoff is linear at one second per consecutive failure, so this window admits the + * first poll, a 1s pause, the second poll and a 2s pause. Anything much above that means + * the loop is not backing off at all. + */ + private static final long OBSERVATION_WINDOW_MILLIS = 2_500; + private static final int MAX_EXPECTED_POLLS = 5; + + private final AtomicInteger watchPolls = new AtomicInteger(); + private final CountDownLatch firstPoll = new CountDownLatch(1); + private HttpServer agentStub; + private KafkaMaaSClientImpl client; + + @BeforeEach + void startAgentStub() throws IOException { + agentStub = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + agentStub.createContext("/api-version", exchange -> respond(exchange, 200, "{\"major\": 2, \"minor\": 8}")); + agentStub.createContext("/api/v2/kafka/topic/watch-create", this::failWatchPoll); + agentStub.start(); + } + + @AfterEach + void stopClientAndStub() { + if (client != null) { + client.close(); + } + agentStub.stop(0); + } + + @Test + void failingWatchPollIsBackedOffInsteadOfHotLooping() { + withProp(Env.PROP_NAMESPACE, NAMESPACE, () -> { + String agentUrl = "http://localhost:" + agentStub.getAddress().getPort(); + withProp(Env.PROP_MAAS_AGENT_URL, agentUrl, () -> { + client = createKafkaClient(agentUrl); + client.watchTopicCreate(WATCHED_TOPIC, addr -> { /* never created in this test */ }); + + assertTrue(firstPoll.await(10, TimeUnit.SECONDS), + "the watch thread never reached the agent stub, so nothing was measured"); + Thread.sleep(OBSERVATION_WINDOW_MILLIS); + + int polls = watchPolls.get(); + // The lower bound matters as much as the upper one: without it the assertion + // would also pass when the loop never ran and nothing was verified. + assertTrue(polls >= 1, "watch loop did not poll at all, the test would pass vacuously"); + assertTrue(polls <= MAX_EXPECTED_POLLS, + "expected the watch loop to back off between failures, but it polled " + polls + + " times in " + OBSERVATION_WINDOW_MILLIS + "ms (limit " + MAX_EXPECTED_POLLS + ")"); + }); + }); + } + + private static KafkaMaaSClientImpl createKafkaClient(String agentUrl) { + System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, agentUrl); + var httpClient = HttpClient.getMaasClient(() -> "faketoken"); + var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); + System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); + + return new KafkaMaaSClientImpl(httpClient, null, new ApiUrlProvider(serverApiVersion, agentUrl)); + } + + /** Answers every poll with 500, the code maas-agent returns when it cannot reach maas-service. */ + private void failWatchPoll(HttpExchange exchange) throws IOException { + watchPolls.incrementAndGet(); + firstPoll.countDown(); + respond(exchange, 500, "{\"error\":\"error proxying request: maas-service unavailable\"}"); + } + + private static void respond(HttpExchange exchange, int status, String body) throws IOException { + exchange.getRequestBody().readAllBytes(); + + byte[] payload = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, payload.length); + try (OutputStream response = exchange.getResponseBody()) { + response.write(payload); + } + } +} diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java new file mode 100644 index 0000000000..26e8490fda --- /dev/null +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -0,0 +1,110 @@ +package com.netcracker.cloud.maas.client.impl.rabbit; + +import com.netcracker.cloud.maas.client.api.Classifier; +import com.netcracker.cloud.maas.client.api.rabbit.VHost; +import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; +import com.netcracker.cloud.maas.client.impl.Env; +import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; +import com.netcracker.cloud.maas.client.impl.http.HttpClient; +import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.junit.jupiter.MockServerExtension; +import org.mockserver.matchers.Times; +import org.mockserver.verify.VerificationTimes; + +import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +@ExtendWith(MockServerExtension.class) +class RabbitFailoverTest { + + private static final String PATH = "/api/v2/rabbit/vhost"; + + @BeforeEach + void reset(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath("/api-version")) + .respond(response().withBody("{\"major\":2, \"minor\": 16}")); + } + + @Test + void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(405) + .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}")); + mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody(""" + { + "cnn": "ampq://rabbit-cluster:4321/maas.core-dev.123456", + "username": "testuser", + "password": "plain:testpassword" + } + """)); + + withProp(Env.PROP_NAMESPACE, "core-dev", () -> + withFastRetries(() -> { + RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); + VHost vhost = client.getOrCreateVirtualHost(new Classifier("commands")); + assertNotNull(vhost); + })); + + mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(500) + .withBody("{\"error\":\"error proxying request: connection refused\"}")); + mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody(""" + { + "cnn": "ampq://rabbit-cluster:4321/maas.core-dev.123456", + "username": "testuser", + "password": "plain:testpassword" + } + """)); + + withProp(Env.PROP_NAMESPACE, "core-dev", () -> + withFastRetries(() -> { + RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); + VHost vhost = client.getOrCreateVirtualHost(new Classifier("commands")); + assertNotNull(vhost); + })); + + mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_400NotRetried(ClientAndServer mockServer) { + mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); + + withProp(Env.PROP_NAMESPACE, "core-dev", () -> + withFastRetries(() -> { + RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); + org.junit.jupiter.api.Assertions.assertThrows(RuntimeException.class, + () -> client.getOrCreateVirtualHost(new Classifier("commands"))); + })); + + mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(1)); + } + + // A short total duration is now the only lever: it bounds both the number of attempts + // and the pauses between them (the cap is derived as a quarter of it). + private static void withFastRetries(Runnable test) { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + } + + private static RabbitMaaSClientImpl createRabbitClient(String agentUrl) { + System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, agentUrl); + var httpClient = HttpClient.getMaasClient(() -> "faketoken"); + var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); + return new RabbitMaaSClientImpl(httpClient, new ApiUrlProvider(serverApiVersion, agentUrl)); + } +} From 280e1f2a0d21177b44d07893f269e006de97535b Mon Sep 17 00:00:00 2001 From: Ksiona Date: Mon, 10 Aug 2026 14:54:15 +0400 Subject: [PATCH 02/24] chore: typo --- .../swagger/config/SwaggerSecurityConfiguratorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java b/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java index 22233fdb47..832f7c38a2 100644 --- a/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java +++ b/core-microservice-framework-extensions/framework-extension-springdoc-swagger/src/test/java/com/netcracker/cloud/frameworkextensions/swagger/config/SwaggerSecurityConfiguratorTest.java @@ -32,7 +32,7 @@ class SwaggerSecurityConfiguratorTest { public static final String IDP_TOKEN_URL = "/api/v1/identity-provider/auth/realms/cloud-common/protocol/openid-connect/token"; private static final String CLIENT_ID = "testuser"; - private static final String CLIENT_SECRET = "tigerword"; + private static final String CLIENT_SECRET = "testpassword"; @Autowired SwaggerSecurityConfigurator configurator; From dd12ae0b336e98025d9e46a115b464286e00deed Mon Sep 17 00:00:00 2001 From: Ksiona Date: Mon, 10 Aug 2026 19:48:03 +0400 Subject: [PATCH 03/24] fix: sonar on old code + retry rule for 405 rc --- maas-client/CHANGELOG.md | 11 +- maas-client/README.md | 34 ++-- .../cloud/maas/client/impl/Env.java | 10 +- .../maas/client/impl/http/HttpExecution.java | 168 ++++++++++++------ .../impl/kafka/KafkaMaaSClientImpl.java | 45 ++++- .../impl/http/HttpExecutionFailoverTest.java | 131 ++++++++++++-- .../impl/rabbit/RabbitFailoverTest.java | 17 +- 7 files changed, 314 insertions(+), 102 deletions(-) diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 5213b9e9cb..cdb1907245 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -3,12 +3,15 @@ ## Unreleased * `Features` - HTTP calls to maas-agent are now retried on retryable status codes, not only on `IOException`. - Retryable: 5xx, 429, **405** and **401**. See "Retry behaviour and configuration" in README for why - the two 4xx codes are included — without them the client does not survive a Postgres leader switchover. + Retryable: 5xx, 429, **405** (only when the body carries a maas-service error) and **401** + (once). See "Retry behaviour and configuration" in README for why the two 4xx codes are + included — without them the client does not survive a Postgres leader switchover. - Backoff is exponential with jitter instead of a fixed 1s delay. - New configuration: `maas.http.retry.max-total-duration-ms` (`60s` by default) — a single setting bounding the whole call. The attempt count and the backoff growth are derived from - it, so there are no separate knobs to keep consistent. + it, so there are no separate knobs to keep consistent. Every attempt is bounded by what is + left of it, so with the defaults the worst case a caller can see is ~60s, not 60s plus one + `maas.http.timeout`. - The Kafka topic `watch-create` long poll no longer goes through the retry policy (`HttpExecution.noRetry()`); its own loop got a linear capped backoff instead, so a down maas-agent is no longer polled in a hot loop. @@ -17,6 +20,8 @@ unexpected 5xx/405/401 threw immediately; it is now retried within the configured limits. - Interrupting a thread during a retry wait now restores the interrupt flag and aborts the loop, instead of swallowing `InterruptedException`. + - `KafkaMaaSClient.watchTopicCreate` throws `IllegalStateException` after `close()`, instead of + registering a callback that can never fire. ## 10.0.0 * `Features` diff --git a/maas-client/README.md b/maas-client/README.md index 4300040e1d..c1df3f53cd 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -68,22 +68,22 @@ setting: the maximum total duration of the call. | `maas.http.timeout` | `30` (seconds) | connect/read/write timeout of a **single** attempt | | `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included | -`max-total-duration-ms` is the only retry knob. The number of attempts and the -growth of the pauses between them are derived from it, so there is nothing to -keep consistent by hand: the first pause is 1s, each next one doubles, and the -cap is a quarter of the total. With the default 60s that gives pauses of -1s, 2s, 4s, 8s, 15s, 15s — roughly six attempts before giving up. +`max-total-duration-ms` is the only retry knob: the attempt count and the pauses +between attempts are derived from it. The first pause is 1s, each next one +doubles, and the cap is a quarter of the total — with the default 60s that gives +1s, 2s, 4s, 8s, 15s, 15s, roughly six attempts when each attempt fails fast. If +attempts hang instead, fewer of them fit into the same budget. Backoff carries ++/-20% jitter so concurrent callers do not retry in lockstep. -The default of 60s is chosen to outlast a database leader switchover, which is -the case these retries exist for, while still failing fast enough for a caller -to react to a real outage. +Each attempt is additionally bounded by what is left of the total duration, so +the worst case a caller sees is the budget itself rather than the budget plus one +`maas.http.timeout`. -Backoff is exponential with +/-20% jitter, so concurrent callers do not retry in -lockstep against a recovering agent. +The 60s default is meant to outlast a database leader switchover while still +failing fast enough to react to a real outage. -The watch endpoint (`watch-create`) is deliberately excluded: it is a long poll with -its own loop, so retrying inside the call would nest two policies and block the watch -for the whole duration. That loop has its own linear, capped backoff instead. +The watch endpoint (`watch-create`) is excluded: it is a long poll with its own +loop and its own backoff. Which responses are retried: @@ -92,12 +92,12 @@ Which responses are retried: | `IOException` | yes | connection refused/reset while the agent is being rescheduled | | 5xx | yes | includes the `500` maas-agent returns when it cannot reach maas-service at all | | 429 | yes | throttling | -| **405** | **yes** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a leader switchover arrives as `405`, not as `5xx` | -| **401** | **yes** | the M2M token is supplied per request, so an expired token or a briefly unavailable token provider clears itself on the next attempt | +| **405** | **only with a maas-service error body** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a switchover arrives as `405`, not as `5xx`. A plain `405` — a route removed on the server, an ingress rejecting the method — is permanent and fails fast | +| **401** | **once** | covers a token that expired in flight. Further attempts re-send the same token, since the supplier cannot be told it was rejected | | other 4xx | no | permanent client errors, failed on the first attempt | -The two 4xx entries are deliberate. Applying the usual "retry 5xx, fail fast on -4xx" rule here means not surviving a database leader switchover. +The two 4xx entries are deliberate: the usual "retry 5xx, fail fast on 4xx" rule +does not survive a database leader switchover here. ## Kafka client usage example All MaaS operations for Kafka is collected in [KafkaMaaSClient](https://github.com/Netcracker/qubership-maas-client/blob/main/client/src/main/java/com/netcracker/cloud/maas/client/api/kafka/KafkaMaaSClient.java). To obtain *new* instance of MaaS Kafka client just call: diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index b3a4f71490..cf01a95a28 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -106,13 +106,9 @@ public static Duration httpTimeout() { } /** - * How long one call to maas-agent may take in total, retries included. This is the - * only retry knob: the number of attempts and the growth of the backoff are derived - * from it, so there is nothing to keep consistent by hand. - *

- * The default of 60s is chosen to outlast a database leader switchover — the case the - * retries exist for — while still failing fast enough for a caller to react to a real - * outage. + * How long one call to maas-agent may take in total, retries included. The only retry + * knob: attempt count and backoff growth are derived from it. The 60s default outlasts + * a database leader switchover. */ public static Duration httpRetryMaxTotalDuration() { return Duration.ofMillis( diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index a2faec2b01..566a64d828 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -9,6 +9,7 @@ import okhttp3.*; import java.io.IOException; +import java.time.Duration; import java.util.*; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -70,9 +71,7 @@ public HttpExecution supressError(int code, Consumer handler) { return this; } - /** - * Performs a single attempt and lets the caller decide what to do with a failure. - */ + /** Performs a single attempt. For callers that own a retry loop, such as a long poll. */ public HttpExecution noRetry() { this.retryEnabled = false; return this; @@ -104,44 +103,70 @@ public Optional sendAndReceive(OmnivoreFunction responseDeseri return sendAndReceive().map(der(responseDeserializer)); } + /** Code carried by every maas-service TMF error envelope. */ + private static final String MAAS_ERROR_CODE = "MAAS-0600"; + /** - * Two 4xx codes are deliberately retryable. Both are transient in this specific chain - * rather than permanent client errors: - *

    - *
  • 405 - maas-service maps PG error 25006 (READ ONLY SQL TRANSACTION) to - * {@code StatusMethodNotAllowed}, so a write against a demoted Patroni node - * during a leader switchover arrives here as 405, not as 5xx.
    • - *
  • 401 - the M2M token is supplied per request by the OkHttp interceptor, so an - * expired token or a briefly unavailable token provider resolves itself on the - * next attempt.
    • - *
+ * Two 4xx are transient here rather than permanent: 405 is how maas-service reports a + * read-only Postgres during a leader switchover, and 401 clears when the token is + * re-supplied on the next attempt. + *

+ * The 405 case is gated on the response body, because a plain 405 — a route removed on + * the server, an ingress rejecting the method — is permanent and must fail fast. */ - private static boolean isRetryableStatus(int code) { + private static boolean isRetryableStatus(int code, String body) { if (code >= 500) { return true; } - return code == 429 || code == 405 || code == 401; + if (code == 429 || code == 401) { + return true; + } + return code == 405 && isDatabaseUnavailable(body); } - /** First backoff pause. Not configurable*/ - private static final long BASE_DELAY_MILLIS = 1_000L; + /** + * Recognises the 405 that maas-service returns for PostgreSQL error 25006, mapped from + * {@code DatabaseIsReadonlyError} / {@code DatabaseIsNotActiveError}. Matched on the + * reason text because the TMF code is the same for every maas-service error. + */ + private static boolean isDatabaseUnavailable(String body) { + if (body == null || !body.contains(MAAS_ERROR_CODE)) { + return false; + } + return body.contains("read-only") || body.contains("not in 'active' mode"); + } /** - * The cap on a single pause is derived from the total duration rather than configured - * separately. A quarter keeps the growth useful at both ends of the range: with the - * default 60s the pauses run 1s, 2s, 4s, 8s, 15s, 15s (~6 attempts), and with a 5s - * total they run 1s, 1.25s, 1.25s, 1.25s (~4 attempts). Either way the caller gets - * several tries without hammering an agent that is coming back up. + * How many times a single call retries a 401. One is enough: it covers a token that + * expired in flight. A token the supplier still considers valid but the server rejects + * comes back identical on every further attempt. */ + static final int MAX_AUTH_RETRIES = 1; + + private int authAttempts = 0; + + private boolean canRetryStatus(int code, String body, long deadlineNanos) { + if (!isRetryableStatus(code, body) || !canRetry(deadlineNanos)) { + return false; + } + if (code == 401) { + return ++authAttempts <= MAX_AUTH_RETRIES; + } + return true; + } + + /** First backoff pause. */ + private static final long BASE_DELAY_MILLIS = 1_000L; + + /** A single pause is capped at this fraction of the total duration. */ private static final int MAX_DELAY_FRACTION_OF_TOTAL = 4; /** - * Delay before jitter: doubles per attempt, capped at the derived maximum. - * Doubling is done in integer arithmetic and saturates at the cap, so a large attempt - * count cannot overflow into a negative delay. + * Delay before jitter: doubles per attempt, capped. Integer arithmetic saturating at + * the cap, so a large attempt count cannot overflow. */ - static long cappedDelayMillis(int attempt) { - long max = Math.max(1L, Env.httpRetryMaxTotalDuration().toMillis() / MAX_DELAY_FRACTION_OF_TOTAL); + static long cappedDelayMillis(int attempt, long maxTotalMillis) { + long max = Math.max(1L, maxTotalMillis / MAX_DELAY_FRACTION_OF_TOTAL); long delay = Math.min(BASE_DELAY_MILLIS, max); for (int i = 1; i < attempt && delay < max; i++) { delay = delay > max / 2 ? max : delay * 2; @@ -150,53 +175,94 @@ static long cappedDelayMillis(int attempt) { } // Exponential backoff with jitter between retries. - private static long backoffMillis(int attempt) { - long capped = cappedDelayMillis(attempt); + private static long backoffMillis(int attempt, long maxTotalMillis) { + long capped = cappedDelayMillis(attempt, maxTotalMillis); double jitterFactor = 0.8 + ThreadLocalRandom.current().nextDouble() * 0.4; return Math.max(1L, (long) (capped * jitterFactor)); } - // The total duration is the only stop condition: there is no attempt counter to - // disagree with it. Callers that own a retry loop opt out entirely via noRetry(). + // The total duration is the only stop condition, unless noRetry() was used. private boolean canRetry(long deadlineNanos) { return retryEnabled && System.nanoTime() < deadlineNanos; } /** - * Waits before the next retry. The delay is clamped to whatever is left of the max - * total duration, otherwise a backoff started just before the deadline would overshoot - * it by up to the configured max delay. Restores the interrupt flag and aborts if - * interrupted. + * Waits before the next retry, clamped to what is left of the total duration so the + * backoff cannot overshoot it. Restores the interrupt flag and aborts if interrupted. */ - private static void sleepBackoff(int attempt, long deadlineNanos) { - long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - if (remainingMillis <= 0) { + private static void sleepBackoff(int attempt, long maxTotalMillis, long deadlineNanos) { + long remaining = remainingMillis(deadlineNanos); + if (remaining <= 0) { return; } try { - Thread.sleep(Math.min(backoffMillis(attempt), remainingMillis)); + Thread.sleep(Math.min(backoffMillis(attempt, maxTotalMillis), remaining)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted while waiting to retry maas-agent request", e); } } - // OkHttp declares Response.body() as nullable; treat a missing body as empty rather - // than dereferencing it. Keeps throwing IOException so the caller's retry loop sees it. + // Response.body() is nullable in OkHttp; a missing body reads as empty. private static String bodyAsString(Response response) throws IOException { ResponseBody body = response.body(); return body == null ? "" : body.string(); } + /** + * Body of a non-2xx response, for the retry decision and the error message. Never throws: + * a body that cannot be read must not turn a permanent status into a retry. + */ + private static String errorBodyOrPlaceholder(Response response) { + try { + return bodyAsString(response); + } catch (IOException e) { + log.debug("Could not read error response body", e); + return ""; + } + } + + private static long remainingMillis(long deadlineNanos) { + return TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + } + + private static String giveUpSuffix(int attempt) { + return attempt == 0 ? "" : ", gave up after " + attempt + " retries"; + } + + /** + * Client for one attempt, bounded by what is left of the total duration. Without it an + * attempt starting just before the deadline still runs for the full + * {@code maas.http.timeout} and the call overruns its budget. + *

+ * Not applied under {@link #noRetry()}: there the caller owns the lifecycle, and the + * watch long poll legitimately runs as long as the budget itself. + */ + private OkHttpClient clientForAttempt(long remainingMs) { + if (!retryEnabled) { + return httpClient; + } + // newBuilder shares the connection pool and dispatcher, so this is cheap + return httpClient.newBuilder() + .callTimeout(Duration.ofMillis(remainingMs)) + .build(); + } + private Optional sendAndReceive() { Request compiledReq = req.build(); log.debug("Send request: {}", compiledReq); long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); - long deadlineNanos = System.nanoTime() + Env.httpRetryMaxTotalDuration().toNanos(); + long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); int attempt = 0; while (true) { - try (Response response = httpClient.newCall(compiledReq).execute()) { + long remainingMs = remainingMillis(deadlineNanos); + if (remainingMs <= 0) { + throw new RuntimeException("Gave up on " + compiledReq + " after " + attempt + + " retries: the " + maxTotalMillis + "ms budget is spent"); + } + + try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { // check response codes against acceptable list log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); @@ -206,17 +272,20 @@ private Optional sendAndReceive() { } if (!expectedCodes.contains(response.code())) { - if (isRetryableStatus(response.code()) && canRetry(deadlineNanos)) { + // read once, without throwing: a body that cannot be read must not turn a + // permanent status into a retry + String errorBody = errorBodyOrPlaceholder(response); + if (canRetryStatus(response.code(), errorBody, deadlineNanos)) { attempt++; log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", response.code(), compiledReq, attempt, maxTotalMillis); - sleepBackoff(attempt, deadlineNanos); + sleepBackoff(attempt, maxTotalMillis, deadlineNanos); continue; } throw new RuntimeException("Unexpected status code " + response.code() + " for request: " + compiledReq - + ", gave up after " + attempt + " retries" - + "\n\tResponse body: " + bodyAsString(response)); + + giveUpSuffix(attempt) + + "\n\tResponse body: " + errorBody); } String body = bodyAsString(response); @@ -224,13 +293,12 @@ private Optional sendAndReceive() { return Optional.of(body); } catch (IOException e) { if (!canRetry(deadlineNanos)) { - throw new RuntimeException("Error executing " + compiledReq - + ", gave up after " + attempt + " retries", e); + throw new RuntimeException("Error executing " + compiledReq + giveUpSuffix(attempt), e); } attempt++; log.warn("Error execute http request: {}, Retry {}, within {}ms total", e.getMessage(), attempt, maxTotalMillis); - sleepBackoff(attempt, deadlineNanos); + sleepBackoff(attempt, maxTotalMillis, deadlineNanos); } } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 09d9cf65f7..0ba55e1fc1 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -46,6 +46,14 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); private volatile boolean closed = false; + /** + * Monitor for parking the watch thread while there is nothing to watch. + *

+ * Not the thread itself: {@link Thread#join()} waits on that same monitor and the JVM + * notifies it when the thread ends, so a notification meant for the watch loop can be + * consumed by a {@code join()} in {@link #close()} and the loop never wakes up. + */ + private final Object watchLock = new Object(); private final Lazy watchThread = new Lazy<>(() -> { Thread exec = new Thread(this::watchTenantCreateTopics, "watchTopicCreate"); exec.setDaemon(true); @@ -92,7 +100,11 @@ public boolean deleteTopic(Classifier classifier) { .sendAndReceive(TopicDeleteResponse.class) .orElse(null); - if (resp != null && !resp.getFailedToDelete().isEmpty()) { + if (resp == null) { + // empty body: nothing was reported as deleted + return false; + } + if (!resp.getFailedToDelete().isEmpty()) { throw new MaaSException("Error delete topic by classifier: %s. Error: %s", classifier, resp.getFailedToDelete().get(0).getMessage()); } @@ -128,16 +140,20 @@ private void watchTenantCreateTopics() { .orElse(Collections.emptyList()); failures = 0; } catch (Exception e) { - // `closed` is the reliable stop signal: an interrupt can be swallowed by - // the HTTP/JSON layers before it reaches us, the flag cannot. - if (closed || Thread.currentThread().isInterrupted()) { + // `closed` is checked too: an interrupt can be swallowed further down + if (closed) { return; // shutting down, not a failure worth reporting } + if (Thread.currentThread().isInterrupted()) { + log.warn("Watch thread interrupted without close(), stopping to watch {}", url, e); + return; + } failures++; log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); if (!sleepWatchBackoff(failures)) { return; // interrupted while backing off } + continue; // `found` is still empty, nothing to deliver } for (TopicInfo addr : found) { @@ -164,11 +180,15 @@ private void watchTenantCreateTopics() { try { log.info("Nothing to watch, sleep thread."); - synchronized (watchThread.get()) { - watchThread.get().wait(); + synchronized (watchLock) { + // guarded wait: a bare wait() would also return on a spurious wakeup + while (!closed && topicCreateListeners.isEmpty()) { + watchLock.wait(); + } } log.info("Woke up!"); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); return; // exit loop } } @@ -194,12 +214,18 @@ private static boolean sleepWatchBackoff(int failures) { @Override public void watchTopicCreate(String name, Consumer callback) { + if (closed) { + // the watch thread has already exited and nothing restarts it, so the callback + // would never fire + throw new IllegalStateException("Client is closed, cannot watch topic: " + name); + } apiProvider.getServerApiVersion().requiresApiVersion(2, 8); log.info("Add watch for topic by: {}, callback: {}", name, callback); topicCreateListeners.computeIfAbsent(new Classifier(name), k -> Collections.synchronizedList(new ArrayList<>())).add(callback); - synchronized (watchThread.get()) { - watchThread.get().notify(); + watchThread.get(); // start the thread if this is the first watch + synchronized (watchLock) { + watchLock.notifyAll(); } } @@ -261,6 +287,9 @@ public List search(SearchCriteria criteria) { @Override public void close() { closed = true; + synchronized (watchLock) { + watchLock.notifyAll(); // release the watch thread if it is parked + } if (watchThread.isInitialized()) { watchThread.get().interrupt(); try { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 2d9ded65f2..51d9d6530c 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -10,7 +10,13 @@ import org.mockserver.matchers.Times; import org.mockserver.verify.VerificationTimes; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -18,6 +24,7 @@ import java.util.concurrent.atomic.AtomicReference; import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -41,7 +48,7 @@ void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { withFastRetries(() -> { Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); assertTrue(body.isPresent()); - assertTrue(body.get().equals("ok")); + assertEquals("ok", body.get()); }); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); @@ -59,16 +66,17 @@ void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { withFastRetries(() -> { Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); assertTrue(body.isPresent()); - assertTrue(body.get().equals("ok")); + assertEquals("ok", body.get()); }); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); } + /** An expired token clears on the next attempt, because the supplier is called again. */ @Test - void testFailover_401TwiceThenSuccess(ClientAndServer mockServer) { + void testFailover_401ThenSuccess(ClientAndServer mockServer) { mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.exactly(2)) + mockServer.when(request().withPath(PATH), Times.exactly(1)) .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(200).withBody("\"ok\"")); @@ -76,10 +84,97 @@ void testFailover_401TwiceThenSuccess(ClientAndServer mockServer) { withFastRetries(() -> { Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); assertTrue(body.isPresent()); - assertTrue(body.get().equals("ok")); + assertEquals("ok", body.get()); }); - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(2)); + } + + /** + * A 401 that keeps coming back means the supplier is handing out a token the server + * rejects, and it has no way of being told so. Further attempts resend the same token, + * so the budget is deliberately tighter than the overall duration: a wrong secret must + * fail fast instead of hanging for the whole minute. + */ + @Test + void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); + + withFastRetries(() -> + assertTrue(assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class) + ).getMessage().contains("401"))); + + mockServer.verify(request().withPath(PATH), + VerificationTimes.exactly(HttpExecution.MAX_AUTH_RETRIES + 1)); + } + + /** + * A 405 without a maas-service error envelope is an ordinary "method not allowed" — + * a route or an ingress rejecting the request — and must fail fast. + */ + @Test + void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(405).withBody("Method Not Allowed")); + + withFastRetries(() -> + assertTrue(assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class) + ).getMessage().contains("405"))); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + + /** + * The per-attempt budget is clamped to what is left of the total duration, so a hanging + * agent cannot stretch the call past it. + */ + @Test + void testMaxTotalDuration_BoundsAHangingAttempt() throws Exception { + // accepts the connection and never answers, unlike a refused connect which fails fast + try (ServerSocket silentServer = new ServerSocket(0)) { + List accepted = Collections.synchronizedList(new ArrayList<>()); + Thread acceptor = new Thread(() -> { + try { + while (!silentServer.isClosed()) { + accepted.add(silentServer.accept()); + } + } catch (IOException e) { + // the socket was closed, the test is over + } + }, "silent-server"); + acceptor.setDaemon(true); + acceptor.start(); + + try { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1000", () -> { + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(Duration.ofMinutes(1)) + .build(); + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + silentServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(client, req).expect(200); + + long start = System.currentTimeMillis(); + assertThrows(RuntimeException.class, () -> execution.sendAndReceive(String.class)); + long elapsedMs = System.currentTimeMillis() - start; + assertTrue(elapsedMs < 20_000, + "expected the call to be bounded by its 1000ms budget rather than by the " + + "one minute read timeout, took " + elapsedMs + "ms"); + }); + } finally { + synchronized (accepted) { + for (Socket socket : accepted) { + socket.close(); + } + } + } + } } @Test @@ -133,16 +228,20 @@ void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws Exception { }); } - // Delay must grow between attempts, not stay flat at the base value. + // Delay must grow between attempts and saturate at a quarter of the total duration. @Test - void testBackoffMillis_GrowsBetweenAttempts() { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "60000", () -> { - long attempt1 = HttpExecution.cappedDelayMillis(1); - long attempt2 = HttpExecution.cappedDelayMillis(2); - long attempt3 = HttpExecution.cappedDelayMillis(3); - assertTrue(attempt2 > attempt1, "expected " + attempt2 + " > " + attempt1); - assertTrue(attempt3 > attempt2, "expected " + attempt3 + " > " + attempt2); - }); + void testBackoffMillis_GrowsAndSaturatesAtTheCap() { + long[] expectedFor60s = {1_000, 2_000, 4_000, 8_000, 15_000, 15_000}; + for (int attempt = 1; attempt <= expectedFor60s.length; attempt++) { + assertEquals(expectedFor60s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 60_000), + "attempt " + attempt + " of a 60s budget"); + } + + long[] expectedFor5s = {1_000, 1_250, 1_250}; + for (int attempt = 1; attempt <= expectedFor5s.length; attempt++) { + assertEquals(expectedFor5s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 5_000), + "attempt " + attempt + " of a 5s budget"); + } } // A tight max total duration must cut the retry loop short well before the attempt count is exhausted. @@ -165,7 +264,7 @@ void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServ // A short total duration is now the only lever: it bounds both the number of attempts // and the pauses between them (the cap is derived as a quarter of it). private static void withFastRetries(Runnable test) { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "5000", test::run); } private static HttpExecution execution(ClientAndServer mockServer) { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java index 26e8490fda..a374ca1ddf 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -7,6 +7,7 @@ import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; import com.netcracker.cloud.maas.client.impl.http.HttpClient; import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -25,13 +26,27 @@ class RabbitFailoverTest { private static final String PATH = "/api/v2/rabbit/vhost"; + private String savedAgentUrl; + @BeforeEach void reset(ClientAndServer mockServer) { + savedAgentUrl = System.getProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); mockServer.reset(); mockServer.when(request().withPath("/api-version")) .respond(response().withBody("{\"major\":2, \"minor\": 16}")); } + // the agent url points at a mock server port that is gone once this class is done, + // so it must not leak into the rest of the JVM + @AfterEach + void restoreAgentUrl() { + if (savedAgentUrl == null) { + System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); + } else { + System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, savedAgentUrl); + } + } + @Test void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) @@ -98,7 +113,7 @@ void testFailover_400NotRetried(ClientAndServer mockServer) { // A short total duration is now the only lever: it bounds both the number of attempts // and the pauses between them (the cap is derived as a quarter of it). private static void withFastRetries(Runnable test) { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "5000", test::run); } private static RabbitMaaSClientImpl createRabbitClient(String agentUrl) { From b511f5c4a5fe8cb8d44bd7a366fbb721a8e4446c Mon Sep 17 00:00:00 2001 From: Ksiona Date: Mon, 10 Aug 2026 22:52:31 +0400 Subject: [PATCH 04/24] fix: sonar issues --- .../cloud/bluegreen/AbstractBGTest.java | 28 +++ maas-client/CHANGELOG.md | 2 + .../cloud/maas/client/api/MaaSException.java | 5 + .../maas/client/api/MaaSHttpException.java | 17 ++ .../cloud/maas/client/impl/Env.java | 7 +- .../maas/client/impl/http/HttpExecution.java | 9 +- .../impl/kafka/KafkaMaaSClientImpl.java | 147 +++++++------ .../impl/http/HttpExecutionFailoverTest.java | 194 ++++++++++-------- .../KafkaMaaSClientWatchBackoffTest.java | 48 ++--- .../impl/rabbit/RabbitFailoverTest.java | 6 +- 10 files changed, 285 insertions(+), 178 deletions(-) create mode 100644 maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java index 47b43c5fa0..8ea5233e4a 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java @@ -7,6 +7,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -29,12 +30,39 @@ class AbstractBGTest { String consulUrl; + private static final Duration NODE_REGISTRATION_TIMEOUT = Duration.ofSeconds(30); + @Container ConsulContainer consulContainer = new ConsulContainer("hashicorp/consul:1.16"); @BeforeEach void before() { consulUrl = String.format("http://%s:%d", consulContainer.getHost(), consulContainer.getMappedPort(8500)); + awaitNodeRegistered(); + } + + /** + * The agent answers on its port before it has registered itself in the catalog, and a session + * cannot be bound to a node that is not there yet: consul replies 500 "Missing node registration". + */ + private void awaitNodeRegistered() { + Instant deadline = Instant.now().plus(NODE_REGISTRATION_TIMEOUT); + String lastSeen = "no response"; + while (Instant.now().isBefore(deadline)) { + try { + String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), + String.class).sendAndGet(); + lastSeen = nodes; + if (nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]")) { + return; + } + } catch (Exception e) { + lastSeen = e.toString(); + } + run(() -> Thread.sleep(100)); + } + throw new IllegalStateException("Consul node was not registered in the catalog within " + + NODE_REGISTRATION_TIMEOUT + ", last response: " + lastSeen); } @SneakyThrows diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index cdb1907245..4ba81264e4 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -22,6 +22,8 @@ instead of swallowing `InterruptedException`. - `KafkaMaaSClient.watchTopicCreate` throws `IllegalStateException` after `close()`, instead of registering a callback that can never fire. + - Failed calls to maas-agent now throw `MaaSHttpException` instead of a bare `RuntimeException`. + It extends `MaaSException`, which is a `RuntimeException`, so existing `catch` blocks keep working. ## 10.0.0 * `Features` diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java index 0308e5b769..c7ff404d39 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java @@ -4,4 +4,9 @@ public class MaaSException extends RuntimeException { public MaaSException(String format, Object...args) { super(String.format(format, args)); } + + /** For subclasses whose message is already built and must not go through String.format. */ + protected MaaSException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java new file mode 100644 index 0000000000..1181b7ec68 --- /dev/null +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java @@ -0,0 +1,17 @@ +package com.netcracker.cloud.maas.client.api; + +/** + * A call to maas-agent that did not succeed: an unexpected status code, or a transport + * failure that outlived the retry budget. The message is taken as is, unlike + * {@link MaaSException}, because it carries request and response text. + */ +public class MaaSHttpException extends MaaSException { + + public MaaSHttpException(String message) { + super(message, (Throwable) null); + } + + public MaaSHttpException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index cf01a95a28..bc1bbe0135 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -211,10 +211,11 @@ private static Optional microProfileConfigOptional(String key) { Object config = getConfig.invoke(null); Method getOptionalValue = config.getClass().getMethod("getOptionalValue", String.class, Class.class); return (Optional) getOptionalValue.invoke(config, key, String.class); - } catch (ClassNotFoundException e) { + } catch (ClassNotFoundException | NoClassDefFoundError e) { + // MicroProfile Config is an optional dependency return Optional.empty(); - } catch (Throwable e) { - log.trace("MicroProfile Config not available or lookup failed for '{}'", key, e); + } catch (Exception e) { + log.trace("MicroProfile Config lookup failed for '{}'", key, e); return Optional.empty(); } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 566a64d828..39d4d6cddc 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.impl.Env; import lombok.extern.slf4j.Slf4j; import okhttp3.*; @@ -199,7 +200,7 @@ private static void sleepBackoff(int attempt, long maxTotalMillis, long deadline Thread.sleep(Math.min(backoffMillis(attempt, maxTotalMillis), remaining)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting to retry maas-agent request", e); + throw new MaaSHttpException("Interrupted while waiting to retry maas-agent request", e); } } @@ -258,7 +259,7 @@ private Optional sendAndReceive() { while (true) { long remainingMs = remainingMillis(deadlineNanos); if (remainingMs <= 0) { - throw new RuntimeException("Gave up on " + compiledReq + " after " + attempt + throw new MaaSHttpException("Gave up on " + compiledReq + " after " + attempt + " retries: the " + maxTotalMillis + "ms budget is spent"); } @@ -282,7 +283,7 @@ private Optional sendAndReceive() { sleepBackoff(attempt, maxTotalMillis, deadlineNanos); continue; } - throw new RuntimeException("Unexpected status code " + response.code() + throw new MaaSHttpException("Unexpected status code " + response.code() + " for request: " + compiledReq + giveUpSuffix(attempt) + "\n\tResponse body: " + errorBody); @@ -293,7 +294,7 @@ private Optional sendAndReceive() { return Optional.of(body); } catch (IOException e) { if (!canRetry(deadlineNanos)) { - throw new RuntimeException("Error executing " + compiledReq + giveUpSuffix(attempt), e); + throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempt), e); } attempt++; log.warn("Error execute http request: {}, Retry {}, within {}ms total", diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 0ba55e1fc1..d369b2e834 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -124,73 +124,104 @@ public void watchTenantTopics(String name, Consumer> callback } private void watchTenantCreateTopics() { - TypeReference> typeRef = new TypeReference<>() { - }; while (!closed) { - int failures = 0; - while (!closed && !topicCreateListeners.isEmpty()) { - String url = apiProvider.getKafkaTopicWatchCreateUrl(watchTimeout); - List found = Collections.emptyList(); - try { - found = httpClient.request(url) - .post(topicCreateListeners.keySet()) - .expect(200) - .noRetry() - .sendAndReceive(typeRef) - .orElse(Collections.emptyList()); - failures = 0; - } catch (Exception e) { - // `closed` is checked too: an interrupt can be swallowed further down - if (closed) { - return; // shutting down, not a failure worth reporting - } - if (Thread.currentThread().isInterrupted()) { - log.warn("Watch thread interrupted without close(), stopping to watch {}", url, e); - return; - } - failures++; - log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); - if (!sleepWatchBackoff(failures)) { - return; // interrupted while backing off - } - continue; // `found` is still empty, nothing to deliver - } - - for (TopicInfo addr : found) { - List> callbacks = topicCreateListeners.remove(addr.getClassifier()); - if (callbacks == null) { - // this is unexpected situation in theory, but with this, code will be a little safer - continue; - } + if (!pollWhileThereIsSomethingToWatch()) { + return; + } + if (closed || !parkUntilThereIsSomethingToWatch()) { + return; + } + } + } - for (Consumer callback : callbacks) { - try { - log.info("Topic create event for {} received, execute callback {}", addr.getClassifier(), callback); - callback.accept(new TopicAddressImpl(addr)); - } catch (Exception e) { - log.error("Error execute callback {}", callback, e); - } - } + /** + * Polls the watch endpoint until nothing is being watched any more. + * + * @return false if the thread must stop + */ + private boolean pollWhileThereIsSomethingToWatch() { + int failures = 0; + while (!closed && !topicCreateListeners.isEmpty()) { + String url = apiProvider.getKafkaTopicWatchCreateUrl(watchTimeout); + List found; + try { + found = poll(url); + failures = 0; + } catch (Exception e) { + // `closed` is checked too: an interrupt can be swallowed further down + if (closed) { + return false; // shutting down, not a failure worth reporting + } + if (Thread.currentThread().isInterrupted()) { + log.warn("Watch thread interrupted without close(), stopping to watch {}", url, e); + return false; + } + failures++; + log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); + if (!sleepWatchBackoff(failures)) { + return false; // interrupted while backing off } + continue; // nothing was received, nothing to deliver } + deliver(found); + } + return true; + } - if (closed) { - return; + /** One long poll for topics created since the previous call. */ + private List poll(String url) { + TypeReference> typeRef = new TypeReference<>() { + }; + return httpClient.request(url) + .post(topicCreateListeners.keySet()) + .expect(200) + .noRetry() + .sendAndReceive(typeRef) + .orElse(Collections.emptyList()); + } + + /** Hands each created topic to the callbacks registered for it, removing them as it goes. */ + private void deliver(List found) { + for (TopicInfo addr : found) { + List> callbacks = topicCreateListeners.remove(addr.getClassifier()); + if (callbacks == null) { + // this is unexpected situation in theory, but with this, code will be a little safer + continue; + } + for (Consumer callback : callbacks) { + notifyCallback(addr, callback); } + } + } - try { - log.info("Nothing to watch, sleep thread."); - synchronized (watchLock) { - // guarded wait: a bare wait() would also return on a spurious wakeup - while (!closed && topicCreateListeners.isEmpty()) { - watchLock.wait(); - } + private void notifyCallback(TopicInfo addr, Consumer callback) { + try { + log.info("Topic create event for {} received, execute callback {}", addr.getClassifier(), callback); + callback.accept(new TopicAddressImpl(addr)); + } catch (Exception e) { + log.error("Error execute callback {}", callback, e); + } + } + + /** + * Parks the thread while no topic is being watched. + * + * @return false if the thread was interrupted and must stop + */ + private boolean parkUntilThereIsSomethingToWatch() { + try { + log.info("Nothing to watch, sleep thread."); + synchronized (watchLock) { + // guarded wait: a bare wait() would also return on a spurious wakeup + while (!closed && topicCreateListeners.isEmpty()) { + watchLock.wait(); } - log.info("Woke up!"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; // exit loop } + log.info("Woke up!"); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 51d9d6530c..4efd3415c5 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -1,5 +1,6 @@ package com.netcracker.cloud.maas.client.impl.http; +import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.impl.Env; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -15,7 +16,6 @@ import java.net.Socket; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; @@ -26,6 +26,7 @@ import static com.netcracker.cloud.maas.client.Utils.withProp; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockserver.model.HttpRequest.request; @@ -102,10 +103,7 @@ void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); - withFastRetries(() -> - assertTrue(assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class) - ).getMessage().contains("401"))); + withFastRetries(() -> assertMessageContains("401", execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(HttpExecution.MAX_AUTH_RETRIES + 1)); @@ -121,10 +119,7 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(405).withBody("Method Not Allowed")); - withFastRetries(() -> - assertTrue(assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class) - ).getMessage().contains("405"))); + withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } @@ -134,46 +129,27 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { * agent cannot stretch the call past it. */ @Test - void testMaxTotalDuration_BoundsAHangingAttempt() throws Exception { + void testMaxTotalDuration_BoundsAHangingAttempt() throws IOException { // accepts the connection and never answers, unlike a refused connect which fails fast try (ServerSocket silentServer = new ServerSocket(0)) { - List accepted = Collections.synchronizedList(new ArrayList<>()); - Thread acceptor = new Thread(() -> { - try { - while (!silentServer.isClosed()) { - accepted.add(silentServer.accept()); - } - } catch (IOException e) { - // the socket was closed, the test is over - } - }, "silent-server"); - acceptor.setDaemon(true); - acceptor.start(); - - try { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1000", () -> { - OkHttpClient client = new OkHttpClient.Builder() - .readTimeout(Duration.ofMinutes(1)) - .build(); - Request.Builder req = new Request.Builder() - .url("http://127.0.0.1:" + silentServer.getLocalPort() + PATH) - .get(); - HttpExecution execution = new HttpExecution(client, req).expect(200); - - long start = System.currentTimeMillis(); - assertThrows(RuntimeException.class, () -> execution.sendAndReceive(String.class)); - long elapsedMs = System.currentTimeMillis() - start; - assertTrue(elapsedMs < 20_000, - "expected the call to be bounded by its 1000ms budget rather than by the " - + "one minute read timeout, took " + elapsedMs + "ms"); - }); - } finally { - synchronized (accepted) { - for (Socket socket : accepted) { - socket.close(); - } - } - } + startAcceptor(silentServer, socket -> { /* hold the connection open and stay silent */ }); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1000", () -> { + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(Duration.ofMinutes(1)) + .build(); + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + silentServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(client, req).expect(200); + + long start = System.currentTimeMillis(); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + long elapsedMs = System.currentTimeMillis() - start; + assertTrue(elapsedMs < 20_000, + "expected the call to be bounded by its 1000ms budget rather than by the " + + "one minute read timeout, took " + elapsedMs + "ms"); + }); } } @@ -183,49 +159,55 @@ void testFailover_400NotRetried(ClientAndServer mockServer) { mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); - withFastRetries(() -> - assertTrue(assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class) - ).getMessage().contains("400"))); + withFastRetries(() -> assertMessageContains("400", execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } @Test - void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws Exception { - // A long total duration keeps the retry wait long enough for the interrupt to land in it. - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "600000", () -> { - OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(Duration.ofMillis(300)) - .build(); - Request.Builder req = new Request.Builder().url("http://127.0.0.1:1/unreachable").get(); - HttpExecution execution = new HttpExecution(client, req); - execution.expect(200); - - AtomicBoolean interruptedAfter = new AtomicBoolean(); - AtomicReference thrown = new AtomicReference<>(); - CountDownLatch started = new CountDownLatch(1); - - Thread worker = new Thread(() -> { - started.countDown(); - try { - execution.sendAndReceive(String.class); - } catch (Throwable t) { - thrown.set(t); - } finally { - interruptedAfter.set(Thread.currentThread().isInterrupted()); - } - }, "http-execution-interrupt-test"); - worker.start(); - - assertTrue(started.await(2, TimeUnit.SECONDS)); - Thread.sleep(500); - worker.interrupt(); - worker.join(5000); - - assertFalse(worker.isAlive(), "worker should abort instead of continuing to retry after interrupt"); - assertTrue(interruptedAfter.get(), "interrupt flag must be restored after an interrupted retry wait"); - }); + void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { + // A server that drops every connection: the attempt fails at once and the loop moves + // into its backoff wait, which is where the interrupt has to land. + try (ServerSocket rudeServer = new ServerSocket(0)) { + CountDownLatch firstAttemptFailed = new CountDownLatch(1); + startAcceptor(rudeServer, socket -> { + socket.close(); + firstAttemptFailed.countDown(); + }); + + // A long total duration keeps the retry wait long enough for the interrupt to land in it. + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "600000", () -> { + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + rudeServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(new OkHttpClient(), req).expect(200); + + AtomicBoolean interruptedAfter = new AtomicBoolean(); + AtomicReference thrown = new AtomicReference<>(); + + Thread worker = new Thread(() -> { + try { + execution.sendAndReceive(String.class); + } catch (Exception e) { + thrown.set(e); + } finally { + interruptedAfter.set(Thread.currentThread().isInterrupted()); + } + }, "http-execution-interrupt-test"); + worker.start(); + + assertTrue(firstAttemptFailed.await(10, TimeUnit.SECONDS), "the first attempt never reached the server"); + worker.interrupt(); + worker.join(10_000); + + assertFalse(worker.isAlive(), "worker should abort instead of continuing to retry after interrupt"); + assertTrue(interruptedAfter.get(), "interrupt flag must be restored after an interrupted retry wait"); + assertInstanceOf(MaaSHttpException.class, thrown.get(), + "the interrupt must surface as a maas exception, not as an unrelated failure"); + assertTrue(thrown.get().getMessage().contains("Interrupted while waiting to retry"), + "unexpected message: " + thrown.get().getMessage()); + }); + } } // Delay must grow between attempts and saturate at a quarter of the total duration. @@ -252,9 +234,9 @@ void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServ .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "200", () -> { + HttpExecution execution = execution(mockServer).expect(200); long start = System.currentTimeMillis(); - assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class)); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); long elapsedMs = System.currentTimeMillis() - start; assertTrue(elapsedMs < 800, "expected retry loop to abort near the 200ms max total duration, took " + elapsedMs + "ms"); @@ -274,4 +256,42 @@ private static HttpExecution execution(ClientAndServer mockServer) { .get(); return new HttpExecution(client, req); } + + private static void assertMessageContains(String expected, HttpExecution execution) { + MaaSHttpException e = assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + assertTrue(e.getMessage().contains(expected), "unexpected message: " + e.getMessage()); + } + + @FunctionalInterface + private interface SocketHandler { + void handle(Socket socket) throws IOException; + } + + /** Serves the socket on a daemon thread until it is closed, then releases what it accepted. */ + private static void startAcceptor(ServerSocket server, SocketHandler handler) { + Thread acceptor = new Thread(() -> { + List accepted = new ArrayList<>(); + try { + while (!server.isClosed()) { + Socket socket = server.accept(); + accepted.add(socket); + handler.handle(socket); + } + } catch (IOException e) { + // the server socket was closed, the test is over + } finally { + accepted.forEach(HttpExecutionFailoverTest::closeQuietly); + } + }, "test-acceptor-" + server.getLocalPort()); + acceptor.setDaemon(true); + acceptor.start(); + } + + private static void closeQuietly(Socket socket) { + try { + socket.close(); + } catch (IOException e) { + // nothing useful to do while tearing a test down + } + } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index a0faf57aeb..c30d1cf83f 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -7,9 +7,11 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -35,16 +37,11 @@ class KafkaMaaSClientWatchBackoffTest { private static final String WATCHED_TOPIC = "orders"; private static final String NAMESPACE = "cloud-dev"; - /** - * The backoff is linear at one second per consecutive failure, so this window admits the - * first poll, a 1s pause, the second poll and a 2s pause. Anything much above that means - * the loop is not backing off at all. - */ - private static final long OBSERVATION_WINDOW_MILLIS = 2_500; - private static final int MAX_EXPECTED_POLLS = 5; + /** Three polls are enough to see the pause between them grow. */ + private static final int OBSERVED_POLLS = 3; - private final AtomicInteger watchPolls = new AtomicInteger(); - private final CountDownLatch firstPoll = new CountDownLatch(1); + private final List pollMillis = Collections.synchronizedList(new ArrayList<>()); + private final CountDownLatch pollsObserved = new CountDownLatch(OBSERVED_POLLS); private HttpServer agentStub; private KafkaMaaSClientImpl client; @@ -72,17 +69,18 @@ void failingWatchPollIsBackedOffInsteadOfHotLooping() { client = createKafkaClient(agentUrl); client.watchTopicCreate(WATCHED_TOPIC, addr -> { /* never created in this test */ }); - assertTrue(firstPoll.await(10, TimeUnit.SECONDS), - "the watch thread never reached the agent stub, so nothing was measured"); - Thread.sleep(OBSERVATION_WINDOW_MILLIS); - - int polls = watchPolls.get(); - // The lower bound matters as much as the upper one: without it the assertion - // would also pass when the loop never ran and nothing was verified. - assertTrue(polls >= 1, "watch loop did not poll at all, the test would pass vacuously"); - assertTrue(polls <= MAX_EXPECTED_POLLS, - "expected the watch loop to back off between failures, but it polled " + polls - + " times in " + OBSERVATION_WINDOW_MILLIS + "ms (limit " + MAX_EXPECTED_POLLS + ")"); + assertTrue(pollsObserved.await(30, TimeUnit.SECONDS), + "the watch loop reached the agent stub only " + pollMillis.size() + + " times out of " + OBSERVED_POLLS + ", so nothing was measured"); + + long firstPause = pollMillis.get(1) - pollMillis.get(0); + long secondPause = pollMillis.get(2) - pollMillis.get(1); + // A hot loop would show pauses near zero; a fixed delay would show two equal ones. + assertTrue(firstPause > 500, + "expected the watch loop to pause after a failure, but it polled again in " + firstPause + "ms"); + assertTrue(secondPause > firstPause, + "expected the pause to grow with consecutive failures, but got " + + firstPause + "ms then " + secondPause + "ms"); }); }); } @@ -93,13 +91,15 @@ private static KafkaMaaSClientImpl createKafkaClient(String agentUrl) { var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); - return new KafkaMaaSClientImpl(httpClient, null, new ApiUrlProvider(serverApiVersion, agentUrl)); + return new KafkaMaaSClientImpl(httpClient, + () -> { throw new UnsupportedOperationException("tenant manager is not used in this test"); }, + new ApiUrlProvider(serverApiVersion, agentUrl)); } /** Answers every poll with 500, the code maas-agent returns when it cannot reach maas-service. */ private void failWatchPoll(HttpExchange exchange) throws IOException { - watchPolls.incrementAndGet(); - firstPoll.countDown(); + pollMillis.add(System.currentTimeMillis()); + pollsObserved.countDown(); respond(exchange, 500, "{\"error\":\"error proxying request: maas-service unavailable\"}"); } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java index a374ca1ddf..048911468c 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -1,6 +1,7 @@ package com.netcracker.cloud.maas.client.impl.rabbit; import com.netcracker.cloud.maas.client.api.Classifier; +import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.api.rabbit.VHost; import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; import com.netcracker.cloud.maas.client.impl.Env; @@ -18,6 +19,7 @@ import static com.netcracker.cloud.maas.client.Utils.withProp; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; @@ -103,8 +105,8 @@ void testFailover_400NotRetried(ClientAndServer mockServer) { withProp(Env.PROP_NAMESPACE, "core-dev", () -> withFastRetries(() -> { RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); - org.junit.jupiter.api.Assertions.assertThrows(RuntimeException.class, - () -> client.getOrCreateVirtualHost(new Classifier("commands"))); + Classifier classifier = new Classifier("commands"); + assertThrows(MaaSHttpException.class, () -> client.getOrCreateVirtualHost(classifier)); })); mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(1)); From e46c5dc23abb152b049cdbe873f96e2235a6d334 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Tue, 11 Aug 2026 14:52:18 +0400 Subject: [PATCH 05/24] fix: total duration counter --- .gitignore | 4 +- maas-client/README.md | 12 +-- .../cloud/maas/client/api/MaaSException.java | 5 ++ .../maas/client/api/MaaSHttpException.java | 2 +- .../cloud/maas/client/impl/Env.java | 27 +++++- .../maas/client/impl/http/HttpExecution.java | 51 ++++++++--- .../impl/kafka/KafkaMaaSClientImpl.java | 23 ++++- .../impl/http/HttpExecutionFailoverTest.java | 85 +++++++++++++++++-- .../KafkaMaaSClientWatchBackoffTest.java | 17 ++++ 9 files changed, 194 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index f278653fb6..b7d5d89334 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ # IDE .idea/ -**/target/ -*.log \ No newline at end of file +target/ +*.log diff --git a/maas-client/README.md b/maas-client/README.md index c1df3f53cd..5457d92243 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -66,24 +66,26 @@ setting: the maximum total duration of the call. | Property | Default | Meaning | |---|---|---| | `maas.http.timeout` | `30` (seconds) | connect/read/write timeout of a **single** attempt | -| `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included | +| `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included. `0` disables retries | `max-total-duration-ms` is the only retry knob: the attempt count and the pauses between attempts are derived from it. The first pause is 1s, each next one doubles, and the cap is a quarter of the total — with the default 60s that gives 1s, 2s, 4s, 8s, 15s, 15s, roughly six attempts when each attempt fails fast. If -attempts hang instead, fewer of them fit into the same budget. Backoff carries +attempts hang instead, fewer of them fit into the same duration. Backoff carries +/-20% jitter so concurrent callers do not retry in lockstep. Each attempt is additionally bounded by what is left of the total duration, so -the worst case a caller sees is the budget itself rather than the budget plus one -`maas.http.timeout`. +the worst case a caller sees is that total duration itself rather than the total +duration plus one `maas.http.timeout`. The 60s default is meant to outlast a database leader switchover while still failing fast enough to react to a real outage. The watch endpoint (`watch-create`) is excluded: it is a long poll with its own -loop and its own backoff. +loop and its own backoff. Its window is derived from `maas.http.timeout` and stays +below it — maas-service holds the request open for the whole window and then answers +with an empty list, which the client has to be able to receive. Which responses are retried: diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java index c7ff404d39..d0a16bfcd9 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java @@ -1,6 +1,11 @@ package com.netcracker.cloud.maas.client.api; public class MaaSException extends RuntimeException { + + /** + * Formats the message. Note that a two-argument call whose second argument is a + * {@code Throwable} binds to the constructor below instead, and is not formatted. + */ public MaaSException(String format, Object...args) { super(String.format(format, args)); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java index 1181b7ec68..69e1a470a7 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java @@ -2,7 +2,7 @@ /** * A call to maas-agent that did not succeed: an unexpected status code, or a transport - * failure that outlived the retry budget. The message is taken as is, unlike + * failure that outlived the configured total duration. The message is taken as is, unlike * {@link MaaSException}, because it carries request and response text. */ public class MaaSHttpException extends MaaSException { diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index bc1bbe0135..762b3a03ea 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -105,20 +105,41 @@ public static Duration httpTimeout() { ); } + static final long DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS = 60_000L; + /** * How long one call to maas-agent may take in total, retries included. The only retry * knob: attempt count and backoff growth are derived from it. The 60s default outlasts * a database leader switchover. + *

+ * Zero disables retries, leaving a single attempt. An unreadable or negative value falls + * back to the default with a warning, rather than failing the call that happens to be first. */ public static Duration httpRetryMaxTotalDuration() { return Duration.ofMillis( stringProperty(PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS) - .map(Long::parseLong) - .filter(ms -> ms > 0) - .orElse(60_000L) + .map(Env::parseRetryDurationMillis) + .orElse(DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS) ); } + private static long parseRetryDurationMillis(String raw) { + long millis; + try { + millis = Long.parseLong(raw.trim()); + } catch (NumberFormatException e) { + log.warn("Ignoring '{}={}': not a number of milliseconds, using {}ms", + PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, raw, DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS); + return DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS; + } + if (millis < 0) { + log.warn("Ignoring '{}={}': negative, using {}ms", + PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, raw, DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS); + return DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS; + } + return millis; + } + public static String url2ws(String url) { return url.replaceAll("^http(s?):", "ws$1:"); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 39d4d6cddc..095ad6c3b8 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -127,14 +127,16 @@ private static boolean isRetryableStatus(int code, String body) { /** * Recognises the 405 that maas-service returns for PostgreSQL error 25006, mapped from - * {@code DatabaseIsReadonlyError} / {@code DatabaseIsNotActiveError}. Matched on the - * reason text because the TMF code is the same for every maas-service error. + * {@code DatabaseIsReadonlyError} ("database is in read-only mode") and + * {@code DatabaseIsNotActiveError} ("database is not in 'active' mode"), both declared in + * maas-service and mapped to 405. */ private static boolean isDatabaseUnavailable(String body) { if (body == null || !body.contains(MAAS_ERROR_CODE)) { return false; } - return body.contains("read-only") || body.contains("not in 'active' mode"); + String reason = body.toLowerCase(Locale.ROOT); + return reason.contains("read-only") || reason.contains("read only") || reason.contains("active"); } /** @@ -146,7 +148,8 @@ private static boolean isDatabaseUnavailable(String body) { private int authAttempts = 0; - private boolean canRetryStatus(int code, String body, long deadlineNanos) { + /** Decides on a retry and counts the 401 attempt, so it is called once per response. */ + private boolean takeRetrySlotFor(int code, String body, long deadlineNanos) { if (!isRetryableStatus(code, body) || !canRetry(deadlineNanos)) { return false; } @@ -234,13 +237,14 @@ private static String giveUpSuffix(int attempt) { /** * Client for one attempt, bounded by what is left of the total duration. Without it an * attempt starting just before the deadline still runs for the full - * {@code maas.http.timeout} and the call overruns its budget. + * {@code maas.http.timeout} and the call overruns its total duration. *

* Not applied under {@link #noRetry()}: there the caller owns the lifecycle, and the - * watch long poll legitimately runs as long as the budget itself. + * watch long poll legitimately runs as long as the total duration itself. */ private OkHttpClient clientForAttempt(long remainingMs) { - if (!retryEnabled) { + if (!retryEnabled || remainingMs <= 0) { + // no retries, or a zero total duration: the single attempt keeps the client's own timeouts return httpClient; } // newBuilder shares the connection pool and dispatcher, so this is cheap @@ -256,11 +260,16 @@ private Optional sendAndReceive() { long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); int attempt = 0; + authAttempts = 0; + // what made the previous attempt fail, so that giving up reports the cause and not + // just the elapsed time + Throwable lastFailure = null; + String lastStatusAndBody = null; while (true) { long remainingMs = remainingMillis(deadlineNanos); - if (remainingMs <= 0) { - throw new MaaSHttpException("Gave up on " + compiledReq + " after " + attempt - + " retries: the " + maxTotalMillis + "ms budget is spent"); + // the total duration bounds retries, not the call: the first attempt always goes out + if (attempt > 0 && remainingMs <= 0) { + throw totalDurationExceeded(compiledReq, attempt, maxTotalMillis, lastStatusAndBody, lastFailure); } try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { @@ -276,8 +285,10 @@ private Optional sendAndReceive() { // read once, without throwing: a body that cannot be read must not turn a // permanent status into a retry String errorBody = errorBodyOrPlaceholder(response); - if (canRetryStatus(response.code(), errorBody, deadlineNanos)) { + if (takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { attempt++; + lastFailure = null; + lastStatusAndBody = "status " + response.code() + ", body: " + errorBody; log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", response.code(), compiledReq, attempt, maxTotalMillis); sleepBackoff(attempt, maxTotalMillis, deadlineNanos); @@ -297,10 +308,28 @@ private Optional sendAndReceive() { throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempt), e); } attempt++; + lastFailure = e; + lastStatusAndBody = null; log.warn("Error execute http request: {}, Retry {}, within {}ms total", e.getMessage(), attempt, maxTotalMillis); sleepBackoff(attempt, maxTotalMillis, deadlineNanos); } } } + + /** + * The usual terminal failure: the backoff is clamped to the time left, so a call that keeps + * failing lands exactly on the deadline. Carries what the last attempt saw, otherwise the + * trace says only that a minute went by. + */ + private static MaaSHttpException totalDurationExceeded(Request req, int attempt, long maxTotalMillis, + String lastStatusAndBody, Throwable lastFailure) { + String message = "Gave up on " + req + " after " + attempt + " retries: ran out of its " + + maxTotalMillis + "ms total duration." + + "\n\tLast attempt: " + (lastStatusAndBody != null ? lastStatusAndBody + : lastFailure != null ? lastFailure.toString() : "unknown"); + return lastFailure != null + ? new MaaSHttpException(message, lastFailure) + : new MaaSHttpException(message); + } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index d369b2e834..d7df069bee 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -23,6 +23,7 @@ import com.netcracker.cloud.maas.client.api.kafka.TopicAddress; import com.netcracker.cloud.maas.client.api.kafka.TopicCreateOptions; import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; +import com.netcracker.cloud.maas.client.impl.Env; import com.netcracker.cloud.maas.client.impl.Lazy; import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicDeleteRequest; import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicDeleteResponse; @@ -40,7 +41,20 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { private final Lazy tenantManagerConnector; private final ApiUrlProvider apiProvider; - private final Duration watchTimeout = Duration.ofSeconds(60); + /** + * How long maas-service is asked to hold a watch poll open. It must stay below the client's + * own read timeout, otherwise every quiet poll dies locally instead of returning an empty + * 200 — which counts as a failure and walks the backoff up to its cap, delaying the next + * real topic-create event. maas-service caps the window at 120s in any case. + */ + private final Duration watchTimeout = watchTimeout(Env.httpTimeout()); + + static Duration watchTimeout(Duration httpTimeout) { + Duration margin = Duration.ofSeconds(5); + Duration window = httpTimeout.minus(margin); + return window.compareTo(margin) < 0 ? margin : window; + } + private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); private static final Duration WATCH_MAX_RETRY_INTERVAL = Duration.ofSeconds(30); // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time @@ -168,15 +182,16 @@ private boolean pollWhileThereIsSomethingToWatch() { return true; } + private static final TypeReference> TOPIC_LIST = new TypeReference<>() { + }; + /** One long poll for topics created since the previous call. */ private List poll(String url) { - TypeReference> typeRef = new TypeReference<>() { - }; return httpClient.request(url) .post(topicCreateListeners.keySet()) .expect(200) .noRetry() - .sendAndReceive(typeRef) + .sendAndReceive(TOPIC_LIST) .orElse(Collections.emptyList()); } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 4efd3415c5..95860eb5f5 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -94,7 +94,7 @@ void testFailover_401ThenSuccess(ClientAndServer mockServer) { /** * A 401 that keeps coming back means the supplier is handing out a token the server * rejects, and it has no way of being told so. Further attempts resend the same token, - * so the budget is deliberately tighter than the overall duration: a wrong secret must + * so retrying it is deliberately capped tighter than the total duration: a wrong secret must * fail fast instead of hanging for the whole minute. */ @Test @@ -125,7 +125,7 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { } /** - * The per-attempt budget is clamped to what is left of the total duration, so a hanging + * Each attempt is clamped to what is left of the total duration, so a hanging * agent cannot stretch the call past it. */ @Test @@ -146,13 +146,62 @@ void testMaxTotalDuration_BoundsAHangingAttempt() throws IOException { long start = System.currentTimeMillis(); assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); long elapsedMs = System.currentTimeMillis() - start; - assertTrue(elapsedMs < 20_000, - "expected the call to be bounded by its 1000ms budget rather than by the " + assertTrue(elapsedMs < 5_000, + "expected the call to be bounded by its 1000ms total duration rather than by the " + "one minute read timeout, took " + elapsedMs + "ms"); }); } } + @Test + void testFailover_429Retried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(1)) + .respond(response().withStatusCode(429).withBody("{\"error\":\"slow down\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertEquals("ok", body.orElseThrow()); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(2)); + } + + /** The watch long poll owns its own loop, so its execution must send the request exactly once. */ + @Test + void testNoRetry_SendsExactlyOneAttempt(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + withFastRetries(() -> { + HttpExecution execution = execution(mockServer).expect(200).noRetry(); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + + /** + * A zero total duration is the config-level off switch: one attempt, no retries, and no + * per-attempt clamp that would cut that attempt short. + */ + @Test + void testZeroTotalDuration_SendsOneAttemptAndDoesNotRetry(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "0", () -> { + HttpExecution execution = execution(mockServer).expect(200); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + @Test void testFailover_400NotRetried(ClientAndServer mockServer) { mockServer.reset(); @@ -210,19 +259,43 @@ void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { } } + /** + * Running out of time is the usual way a failover call ends, so the exception has to say what + * kept failing. Without the cause the trace shows only that a minute went by. + */ + @Test + void testTotalDurationExceeded_CarriesTheLastFailureAsCause() throws IOException { + try (ServerSocket rudeServer = new ServerSocket(0)) { + startAcceptor(rudeServer, Socket::close); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1500", () -> { + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + rudeServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(new OkHttpClient(), req).expect(200); + + MaaSHttpException e = assertThrows(MaaSHttpException.class, + () -> execution.sendAndReceive(String.class)); + assertTrue(e.getMessage().contains("ran out of its"), "unexpected message: " + e.getMessage()); + assertInstanceOf(IOException.class, e.getCause(), + "the transport failure that consumed the time must be the cause"); + }); + } + } + // Delay must grow between attempts and saturate at a quarter of the total duration. @Test void testBackoffMillis_GrowsAndSaturatesAtTheCap() { long[] expectedFor60s = {1_000, 2_000, 4_000, 8_000, 15_000, 15_000}; for (int attempt = 1; attempt <= expectedFor60s.length; attempt++) { assertEquals(expectedFor60s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 60_000), - "attempt " + attempt + " of a 60s budget"); + "attempt " + attempt + " of a 60s total duration"); } long[] expectedFor5s = {1_000, 1_250, 1_250}; for (int attempt = 1; attempt <= expectedFor5s.length; attempt++) { assertEquals(expectedFor5s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 5_000), - "attempt " + attempt + " of a 5s budget"); + "attempt " + attempt + " of a 5s total duration"); } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index c30d1cf83f..70f213eb59 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -1,9 +1,11 @@ package com.netcracker.cloud.maas.client.impl.kafka; import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.time.Duration; import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; @@ -61,6 +63,21 @@ void stopClientAndStub() { agentStub.stop(0); } + /** + * maas-service holds a watch poll open for the whole requested window and then answers 200 + * with an empty list. If the window outlasts the client read timeout, that answer never + * arrives: every quiet poll fails locally, walks the backoff up to its 30s cap and delays + * the next real topic-create event. + */ + @Test + void watchWindowStaysBelowTheReadTimeout() { + assertTrue(KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)).compareTo(Duration.ofSeconds(30)) < 0, + "the watch window must leave the read timeout room to receive the answer"); + assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30))); + // a read timeout too small to leave a margin still yields a usable window + assertEquals(Duration.ofSeconds(5), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(2))); + } + @Test void failingWatchPollIsBackedOffInsteadOfHotLooping() { withProp(Env.PROP_NAMESPACE, NAMESPACE, () -> { From 06e3bcc7aa8d0ff914c7e46b0460538f186d2756 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Wed, 12 Aug 2026 10:39:06 +0400 Subject: [PATCH 06/24] fix: sonar issues and documentation --- .../blue-green-state-monitor-java/pom.xml | 5 + .../cloud/bluegreen/AbstractBGTest.java | 32 +++--- maas-client/CHANGELOG.md | 6 + maas-client/README.md | 3 +- .../maas/client/impl/http/HttpExecution.java | 106 +++++++++++------- .../impl/kafka/KafkaMaaSClientImpl.java | 14 ++- .../impl/http/HttpExecutionFailoverTest.java | 16 +++ .../KafkaMaaSClientWatchBackoffTest.java | 15 ++- 8 files changed, 132 insertions(+), 65 deletions(-) diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml index 71fab48ee4..9d09890e05 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml @@ -56,6 +56,11 @@ junit-jupiter test + + org.awaitility + awaitility + test + com.squareup.okhttp3 okhttp diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java index 8ea5233e4a..adf79c0b80 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java @@ -2,6 +2,7 @@ import com.netcracker.cloud.bluegreen.impl.http.HttpClientAdapter; import lombok.SneakyThrows; +import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeEach; import org.testcontainers.consul.ConsulContainer; import org.testcontainers.junit.jupiter.Container; @@ -14,8 +15,9 @@ import java.util.concurrent.Callable; import java.util.function.Supplier; +/** Shared fixture: a Consul container per test method, plus small waiting helpers. */ @Testcontainers -class AbstractBGTest { +abstract class AbstractBGTest { String ns1 = "ns-1"; String ns2 = "ns-2"; @@ -46,23 +48,17 @@ void before() { * cannot be bound to a node that is not there yet: consul replies 500 "Missing node registration". */ private void awaitNodeRegistered() { - Instant deadline = Instant.now().plus(NODE_REGISTRATION_TIMEOUT); - String lastSeen = "no response"; - while (Instant.now().isBefore(deadline)) { - try { - String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), - String.class).sendAndGet(); - lastSeen = nodes; - if (nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]")) { - return; - } - } catch (Exception e) { - lastSeen = e.toString(); - } - run(() -> Thread.sleep(100)); - } - throw new IllegalStateException("Consul node was not registered in the catalog within " - + NODE_REGISTRATION_TIMEOUT + ", last response: " + lastSeen); + Awaitility.await("consul node registered in the catalog") + .atMost(NODE_REGISTRATION_TIMEOUT) + .pollInterval(Duration.ofMillis(100)) + .ignoreExceptions() + .until(this::catalogHasNodes); + } + + private boolean catalogHasNodes() { + String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), + String.class).sendAndGet(); + return nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]"); } @SneakyThrows diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 4ba81264e4..ca5ebb86ea 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -24,6 +24,12 @@ registering a callback that can never fire. - Failed calls to maas-agent now throw `MaaSHttpException` instead of a bare `RuntimeException`. It extends `MaaSException`, which is a `RuntimeException`, so existing `catch` blocks keep working. + Note the widening: `catch (MaaSException)` used to mean a MaaS business error and now also + catches transport failures, such as the agent being unreachable for the whole minute. + - `maas.http.retry.max-total-duration-ms=0` disables retries, leaving a single attempt. An + unreadable or negative value logs a warning and falls back to the 60s default. + - The Kafka watch poll window is derived from `maas.http.timeout` (25s with the defaults) instead + of a fixed 60s that outlasted the read timeout, so tuning `maas.http.timeout` now also moves it. ## 10.0.0 * `Features` diff --git a/maas-client/README.md b/maas-client/README.md index 5457d92243..a075de24bb 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -85,7 +85,8 @@ failing fast enough to react to a real outage. The watch endpoint (`watch-create`) is excluded: it is a long poll with its own loop and its own backoff. Its window is derived from `maas.http.timeout` and stays below it — maas-service holds the request open for the whole window and then answers -with an empty list, which the client has to be able to receive. +with an empty list, which the client has to be able to receive. With the default 30s +timeout the window is 25s. Which responses are retried: diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 095ad6c3b8..fd29f63d67 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -136,7 +136,8 @@ private static boolean isDatabaseUnavailable(String body) { return false; } String reason = body.toLowerCase(Locale.ROOT); - return reason.contains("read-only") || reason.contains("read only") || reason.contains("active"); + return reason.contains("read-only") || reason.contains("read only") + || reason.contains("not in 'active' mode"); } /** @@ -259,17 +260,12 @@ private Optional sendAndReceive() { long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); - int attempt = 0; + Attempts attempts = new Attempts(); authAttempts = 0; - // what made the previous attempt fail, so that giving up reports the cause and not - // just the elapsed time - Throwable lastFailure = null; - String lastStatusAndBody = null; while (true) { long remainingMs = remainingMillis(deadlineNanos); - // the total duration bounds retries, not the call: the first attempt always goes out - if (attempt > 0 && remainingMs <= 0) { - throw totalDurationExceeded(compiledReq, attempt, maxTotalMillis, lastStatusAndBody, lastFailure); + if (attempts.outOfTime(remainingMs)) { + throw totalDurationExceeded(compiledReq, attempts, maxTotalMillis); } try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { @@ -282,22 +278,8 @@ private Optional sendAndReceive() { } if (!expectedCodes.contains(response.code())) { - // read once, without throwing: a body that cannot be read must not turn a - // permanent status into a retry - String errorBody = errorBodyOrPlaceholder(response); - if (takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { - attempt++; - lastFailure = null; - lastStatusAndBody = "status " + response.code() + ", body: " + errorBody; - log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", - response.code(), compiledReq, attempt, maxTotalMillis); - sleepBackoff(attempt, maxTotalMillis, deadlineNanos); - continue; - } - throw new MaaSHttpException("Unexpected status code " + response.code() - + " for request: " + compiledReq - + giveUpSuffix(attempt) - + "\n\tResponse body: " + errorBody); + retryOrFail(response, compiledReq, attempts, deadlineNanos, maxTotalMillis); + continue; } String body = bodyAsString(response); @@ -305,31 +287,79 @@ private Optional sendAndReceive() { return Optional.of(body); } catch (IOException e) { if (!canRetry(deadlineNanos)) { - throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempt), e); + throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempts.count), e); } - attempt++; - lastFailure = e; - lastStatusAndBody = null; + attempts.afterTransportError(e); log.warn("Error execute http request: {}, Retry {}, within {}ms total", - e.getMessage(), attempt, maxTotalMillis); - sleepBackoff(attempt, maxTotalMillis, deadlineNanos); + e.getMessage(), attempts.count, maxTotalMillis); + sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); } } } + /** + * Handles a status the caller did not expect: waits before the next attempt, or throws when + * the status is terminal. + */ + private void retryOrFail(Response response, Request compiledReq, Attempts attempts, + long deadlineNanos, long maxTotalMillis) { + // read once, without throwing: a body that cannot be read must not turn a permanent + // status into a retry + String errorBody = errorBodyOrPlaceholder(response); + if (!takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { + throw new MaaSHttpException("Unexpected status code " + response.code() + + " for request: " + compiledReq + + giveUpSuffix(attempts.count) + + "\n\tResponse body: " + errorBody); + } + attempts.afterStatus(response.code(), errorBody); + log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", + response.code(), compiledReq, attempts.count, maxTotalMillis); + sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); + } + + /** How many attempts went out and what the last one failed with. */ + private static final class Attempts { + private int count; + private Throwable lastFailure; + private String lastStatusAndBody; + + /** The total duration bounds retries, not the call: the first attempt always goes out. */ + boolean outOfTime(long remainingMs) { + return count > 0 && remainingMs <= 0; + } + + void afterStatus(int code, String body) { + count++; + lastFailure = null; + lastStatusAndBody = "status " + code + ", body: " + body; + } + + void afterTransportError(IOException e) { + count++; + lastFailure = e; + lastStatusAndBody = null; + } + + String describeLast() { + if (lastStatusAndBody != null) { + return lastStatusAndBody; + } + return lastFailure != null ? lastFailure.toString() : "unknown"; + } + } + /** * The usual terminal failure: the backoff is clamped to the time left, so a call that keeps * failing lands exactly on the deadline. Carries what the last attempt saw, otherwise the * trace says only that a minute went by. */ - private static MaaSHttpException totalDurationExceeded(Request req, int attempt, long maxTotalMillis, - String lastStatusAndBody, Throwable lastFailure) { - String message = "Gave up on " + req + " after " + attempt + " retries: ran out of its " + private static MaaSHttpException totalDurationExceeded(Request req, Attempts attempts, long maxTotalMillis) { + String message = "Gave up on " + req + " after " + attempts.count + " retries: ran out of its " + maxTotalMillis + "ms total duration." - + "\n\tLast attempt: " + (lastStatusAndBody != null ? lastStatusAndBody - : lastFailure != null ? lastFailure.toString() : "unknown"); - return lastFailure != null - ? new MaaSHttpException(message, lastFailure) + + "\n\tLast attempt: " + attempts.describeLast(); + return attempts.lastFailure != null + ? new MaaSHttpException(message, attempts.lastFailure) : new MaaSHttpException(message); } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index d7df069bee..565bccebc0 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -49,10 +49,18 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { */ private final Duration watchTimeout = watchTimeout(Env.httpTimeout()); + /** Largest gap left between the watch window and the read timeout. */ + private static final long MAX_WATCH_MARGIN_SECONDS = 5; + + /** + * The margin is clamped rather than subtracted outright, so that a small read timeout + * narrows the window instead of pushing it past the timeout. Whole seconds, because that + * is how the window travels in the query string. + */ static Duration watchTimeout(Duration httpTimeout) { - Duration margin = Duration.ofSeconds(5); - Duration window = httpTimeout.minus(margin); - return window.compareTo(margin) < 0 ? margin : window; + long timeoutSeconds = httpTimeout.getSeconds(); + long marginSeconds = Math.min(MAX_WATCH_MARGIN_SECONDS, timeoutSeconds / 2); + return Duration.ofSeconds(Math.max(1, timeoutSeconds - marginSeconds)); } private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 95860eb5f5..fb325707d1 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -124,6 +124,22 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } + /** + * The envelope alone does not make a 405 transient: every maas-service error carries the same + * code, so the reason has to name the read-only database and not merely contain its words. + */ + @Test + void testFailover_405WithUnrelatedMaasReasonNotRetried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(405) + .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}")); + + withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + /** * Each attempt is clamped to what is left of the total duration, so a hanging * agent cannot stretch the call past it. diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index 70f213eb59..f5521af4ef 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -71,11 +71,16 @@ void stopClientAndStub() { */ @Test void watchWindowStaysBelowTheReadTimeout() { - assertTrue(KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)).compareTo(Duration.ofSeconds(30)) < 0, - "the watch window must leave the read timeout room to receive the answer"); - assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30))); - // a read timeout too small to leave a margin still yields a usable window - assertEquals(Duration.ofSeconds(5), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(2))); + // the invariant, checked across the range rather than at one point + for (long readTimeoutSeconds : new long[]{2, 5, 6, 10, 30, 60, 120}) { + Duration window = KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(readTimeoutSeconds)); + assertTrue(window.getSeconds() < readTimeoutSeconds, + "a " + readTimeoutSeconds + "s read timeout must leave room for the answer, got " + window); + assertTrue(window.getSeconds() >= 1, + "the window travels in whole seconds, so it must not round down to zero: " + window); + } + assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)), + "the default read timeout should keep the full margin"); } @Test From 8f508a18b378c5a35a14235400d6f9115adcbe11 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Wed, 12 Aug 2026 13:23:34 +0400 Subject: [PATCH 07/24] fix: added parametrized test for several similar ones --- .../impl/http/HttpExecutionFailoverTest.java | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index fb325707d1..685d094085 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -6,6 +6,9 @@ import okhttp3.Request; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockserver.integration.ClientAndServer; import org.mockserver.junit.jupiter.MockServerExtension; import org.mockserver.matchers.Times; @@ -22,6 +25,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import static com.netcracker.cloud.maas.client.Utils.withProp; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -29,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; @@ -109,33 +114,29 @@ void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { VerificationTimes.exactly(HttpExecution.MAX_AUTH_RETRIES + 1)); } - /** - * A 405 without a maas-service error envelope is an ordinary "method not allowed" — - * a route or an ingress rejecting the request — and must fail fast. - */ - @Test - void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(405).withBody("Method Not Allowed")); - - withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); - - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + /** Responses that are permanent, so the call must fail on its first attempt. */ + static Stream permanentResponses() { + return Stream.of( + arguments("a plain client error", 400, "{\"error\":\"bad request\"}"), + // 405 is transient only for a read-only database; a route removed on the server + // or an ingress rejecting the method is not + arguments("405 without a maas-service envelope", 405, "Method Not Allowed"), + // every maas-service error carries MAAS-0600, so the envelope alone means nothing: + // the reason has to name the read-only database, not merely contain its words + arguments("405 whose maas-service reason is unrelated", 405, + "{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}") + ); } - /** - * The envelope alone does not make a 405 transient: every maas-service error carries the same - * code, so the reason has to name the read-only database and not merely contain its words. - */ - @Test - void testFailover_405WithUnrelatedMaasReasonNotRetried(ClientAndServer mockServer) { + @ParameterizedTest(name = "{0} is not retried") + @MethodSource("permanentResponses") + void testFailover_PermanentResponseNotRetried(String description, int status, String body, + ClientAndServer mockServer) { mockServer.reset(); mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(405) - .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}")); + .respond(response().withStatusCode(status).withBody(body)); - withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); + withFastRetries(() -> assertMessageContains(String.valueOf(status), execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } @@ -218,17 +219,6 @@ void testZeroTotalDuration_SendsOneAttemptAndDoesNotRetry(ClientAndServer mockSe mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } - @Test - void testFailover_400NotRetried(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); - - withFastRetries(() -> assertMessageContains("400", execution(mockServer).expect(200))); - - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); - } - @Test void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { // A server that drops every connection: the attempt fails at once and the loop moves From 0f507dfe87b0cd949b9e14f872ed31a465fdc1bf Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 11:24:45 +0400 Subject: [PATCH 08/24] chore: refactoring, failsafe package were added --- .../blue-green-state-monitor-java/pom.xml | 5 - .../cloud/bluegreen/AbstractBGTest.java | 27 +- maas-client/client/pom.xml | 4 + .../maas/client/impl/http/HttpExecution.java | 249 ++++++++---------- .../impl/http/HttpExecutionFailoverTest.java | 16 -- maas-client/pom.xml | 6 + 6 files changed, 118 insertions(+), 189 deletions(-) diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml index 9d09890e05..71fab48ee4 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml @@ -56,11 +56,6 @@ junit-jupiter test - - org.awaitility - awaitility - test - com.squareup.okhttp3 okhttp diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java index adf79c0b80..bae06a4efd 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java @@ -2,22 +2,20 @@ import com.netcracker.cloud.bluegreen.impl.http.HttpClientAdapter; import lombok.SneakyThrows; -import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeEach; import org.testcontainers.consul.ConsulContainer; +import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.concurrent.Callable; import java.util.function.Supplier; -/** Shared fixture: a Consul container per test method, plus small waiting helpers. */ @Testcontainers -abstract class AbstractBGTest { +class AbstractBGTest { String ns1 = "ns-1"; String ns2 = "ns-2"; @@ -32,33 +30,12 @@ abstract class AbstractBGTest { String consulUrl; - private static final Duration NODE_REGISTRATION_TIMEOUT = Duration.ofSeconds(30); - @Container ConsulContainer consulContainer = new ConsulContainer("hashicorp/consul:1.16"); @BeforeEach void before() { consulUrl = String.format("http://%s:%d", consulContainer.getHost(), consulContainer.getMappedPort(8500)); - awaitNodeRegistered(); - } - - /** - * The agent answers on its port before it has registered itself in the catalog, and a session - * cannot be bound to a node that is not there yet: consul replies 500 "Missing node registration". - */ - private void awaitNodeRegistered() { - Awaitility.await("consul node registered in the catalog") - .atMost(NODE_REGISTRATION_TIMEOUT) - .pollInterval(Duration.ofMillis(100)) - .ignoreExceptions() - .until(this::catalogHasNodes); - } - - private boolean catalogHasNodes() { - String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), - String.class).sendAndGet(); - return nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]"); } @SneakyThrows diff --git a/maas-client/client/pom.xml b/maas-client/client/pom.xml index 2064105fd8..9a7fe7e62f 100644 --- a/maas-client/client/pom.xml +++ b/maas-client/client/pom.xml @@ -26,6 +26,10 @@ lombok provided + + dev.failsafe + failsafe + com.squareup.okhttp3 diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index fd29f63d67..0392146641 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -6,14 +6,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.impl.Env; +import dev.failsafe.ExecutionContext; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; +import dev.failsafe.RetryPolicyBuilder; import lombok.extern.slf4j.Slf4j; import okhttp3.*; import java.io.IOException; import java.time.Duration; import java.util.*; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -149,64 +152,15 @@ private static boolean isDatabaseUnavailable(String body) { private int authAttempts = 0; - /** Decides on a retry and counts the 401 attempt, so it is called once per response. */ - private boolean takeRetrySlotFor(int code, String body, long deadlineNanos) { - if (!isRetryableStatus(code, body) || !canRetry(deadlineNanos)) { - return false; - } - if (code == 401) { - return ++authAttempts <= MAX_AUTH_RETRIES; - } - return true; + /** Asked once per failed attempt, so the 401 is counted here. */ + private boolean worthAnotherAttempt(RetryableStatus status) { + return status.code != 401 || ++authAttempts <= MAX_AUTH_RETRIES; } - /** First backoff pause. */ - private static final long BASE_DELAY_MILLIS = 1_000L; - - /** A single pause is capped at this fraction of the total duration. */ + /** First backoff pause, and the fraction of the total duration a single pause may reach. */ + private static final Duration BASE_DELAY = Duration.ofSeconds(1); private static final int MAX_DELAY_FRACTION_OF_TOTAL = 4; - - /** - * Delay before jitter: doubles per attempt, capped. Integer arithmetic saturating at - * the cap, so a large attempt count cannot overflow. - */ - static long cappedDelayMillis(int attempt, long maxTotalMillis) { - long max = Math.max(1L, maxTotalMillis / MAX_DELAY_FRACTION_OF_TOTAL); - long delay = Math.min(BASE_DELAY_MILLIS, max); - for (int i = 1; i < attempt && delay < max; i++) { - delay = delay > max / 2 ? max : delay * 2; - } - return delay; - } - - // Exponential backoff with jitter between retries. - private static long backoffMillis(int attempt, long maxTotalMillis) { - long capped = cappedDelayMillis(attempt, maxTotalMillis); - double jitterFactor = 0.8 + ThreadLocalRandom.current().nextDouble() * 0.4; - return Math.max(1L, (long) (capped * jitterFactor)); - } - - // The total duration is the only stop condition, unless noRetry() was used. - private boolean canRetry(long deadlineNanos) { - return retryEnabled && System.nanoTime() < deadlineNanos; - } - - /** - * Waits before the next retry, clamped to what is left of the total duration so the - * backoff cannot overshoot it. Restores the interrupt flag and aborts if interrupted. - */ - private static void sleepBackoff(int attempt, long maxTotalMillis, long deadlineNanos) { - long remaining = remainingMillis(deadlineNanos); - if (remaining <= 0) { - return; - } - try { - Thread.sleep(Math.min(backoffMillis(attempt, maxTotalMillis), remaining)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new MaaSHttpException("Interrupted while waiting to retry maas-agent request", e); - } - } + private static final double JITTER = 0.2; // Response.body() is nullable in OkHttp; a missing body reads as empty. private static String bodyAsString(Response response) throws IOException { @@ -227,10 +181,6 @@ private static String errorBodyOrPlaceholder(Response response) { } } - private static long remainingMillis(long deadlineNanos) { - return TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - } - private static String giveUpSuffix(int attempt) { return attempt == 0 ? "" : ", gave up after " + attempt + " retries"; } @@ -254,112 +204,125 @@ private OkHttpClient clientForAttempt(long remainingMs) { .build(); } + /** + * Backoff, jitter, attempt counting and the overall deadline belong to the retry policy; + * what is left here is what one attempt is and which of its outcomes is worth repeating. + */ private Optional sendAndReceive() { Request compiledReq = req.build(); log.debug("Send request: {}", compiledReq); long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); - long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); - Attempts attempts = new Attempts(); + if (!retryEnabled || maxTotalMillis <= 0) { + return attemptOnce(compiledReq); + } authAttempts = 0; - while (true) { - long remainingMs = remainingMillis(deadlineNanos); - if (attempts.outOfTime(remainingMs)) { - throw totalDurationExceeded(compiledReq, attempts, maxTotalMillis); + try { + return Failsafe.with(retryPolicy(compiledReq, maxTotalMillis)).get(context -> + attempt(compiledReq, maxTotalMillis - context.getElapsedTime().toMillis(), context)); + } catch (RetryableStatus e) { + throw exhausted(compiledReq, maxTotalMillis, e.describe(), null); + } catch (FailsafeException e) { + if (e.getCause() instanceof InterruptedException interrupted) { + // Failsafe restores the flag; the call still has to abort rather than retry + Thread.currentThread().interrupt(); + throw new MaaSHttpException("Interrupted while waiting to retry maas-agent request", interrupted); } + throw exhausted(compiledReq, maxTotalMillis, String.valueOf(e.getCause()), e.getCause()); + } + } - try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { - // check response codes against acceptable list - log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); + private RetryPolicy> retryPolicy(Request compiledReq, long maxTotalMillis) { + Duration maxDelay = Duration.ofMillis(Math.max(1, maxTotalMillis / MAX_DELAY_FRACTION_OF_TOTAL)); + RetryPolicyBuilder> policy = RetryPolicy.builder(); + if (BASE_DELAY.compareTo(maxDelay) < 0) { + policy.withBackoff(BASE_DELAY, maxDelay, 2.0); + } else { + // a total duration too short for the pause to grow leaves one pause of the capped size + policy.withDelay(maxDelay); + } + return policy + .handle(IOException.class) + .handleIf((ignored, failure) -> + failure instanceof RetryableStatus status && worthAnotherAttempt(status)) + .withJitter(JITTER) + .withMaxAttempts(-1) + .withMaxDuration(Duration.ofMillis(maxTotalMillis)) + .onRetry(event -> log.warn("Retrying request: {}. Attempt {} failed with {}, within {}ms total", + compiledReq, event.getAttemptCount(), describe(event.getLastException()), maxTotalMillis)) + .build(); + } - if (errorHandler.containsKey(response.code())) { - errorHandler.get(response.code()).accept(bodyAsString(response)); - return Optional.empty(); - } + /** One request/response exchange. Throws {@link RetryableStatus} for an outcome worth repeating. */ + private Optional attempt(Request compiledReq, long remainingMs, + ExecutionContext> context) throws IOException { + try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { + // check response codes against acceptable list + log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); - if (!expectedCodes.contains(response.code())) { - retryOrFail(response, compiledReq, attempts, deadlineNanos, maxTotalMillis); - continue; - } + if (errorHandler.containsKey(response.code())) { + errorHandler.get(response.code()).accept(bodyAsString(response)); + return Optional.empty(); + } - String body = bodyAsString(response); - log.debug("Response body: {}", body); - return Optional.of(body); - } catch (IOException e) { - if (!canRetry(deadlineNanos)) { - throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempts.count), e); + if (!expectedCodes.contains(response.code())) { + // read once, without throwing: a body that cannot be read must not turn a permanent + // status into a retry + String errorBody = errorBodyOrPlaceholder(response); + if (isRetryableStatus(response.code(), errorBody)) { + throw new RetryableStatus(response.code(), errorBody); } - attempts.afterTransportError(e); - log.warn("Error execute http request: {}, Retry {}, within {}ms total", - e.getMessage(), attempts.count, maxTotalMillis); - sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); + throw new MaaSHttpException("Unexpected status code " + response.code() + + " for request: " + compiledReq + + giveUpSuffix(context == null ? 0 : context.getAttemptCount()) + + "\n\tResponse body: " + errorBody); } + + String body = bodyAsString(response); + log.debug("Response body: {}", body); + return Optional.of(body); } } - /** - * Handles a status the caller did not expect: waits before the next attempt, or throws when - * the status is terminal. - */ - private void retryOrFail(Response response, Request compiledReq, Attempts attempts, - long deadlineNanos, long maxTotalMillis) { - // read once, without throwing: a body that cannot be read must not turn a permanent - // status into a retry - String errorBody = errorBodyOrPlaceholder(response); - if (!takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { - throw new MaaSHttpException("Unexpected status code " + response.code() - + " for request: " + compiledReq - + giveUpSuffix(attempts.count) - + "\n\tResponse body: " + errorBody); + /** The {@link #noRetry()} path, and a total duration configured to zero. */ + private Optional attemptOnce(Request compiledReq) { + try { + return attempt(compiledReq, 0, null); + } catch (RetryableStatus e) { + throw new MaaSHttpException("Unexpected status code " + e.code + + " for request: " + compiledReq + "\n\tResponse body: " + e.body); + } catch (IOException e) { + throw new MaaSHttpException("Error executing " + compiledReq, e); } - attempts.afterStatus(response.code(), errorBody); - log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", - response.code(), compiledReq, attempts.count, maxTotalMillis); - sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); } - /** How many attempts went out and what the last one failed with. */ - private static final class Attempts { - private int count; - private Throwable lastFailure; - private String lastStatusAndBody; + private static String describe(Throwable failure) { + return failure instanceof RetryableStatus status ? status.describe() : String.valueOf(failure); + } - /** The total duration bounds retries, not the call: the first attempt always goes out. */ - boolean outOfTime(long remainingMs) { - return count > 0 && remainingMs <= 0; - } + /** + * The usual terminal failure: a call that keeps failing lands on the deadline. Carries what the + * last attempt saw, otherwise the trace says only that a minute went by. + */ + private static MaaSHttpException exhausted(Request req, long maxTotalMillis, String lastAttempt, Throwable cause) { + String message = "Gave up on " + req + ": ran out of its " + maxTotalMillis + + "ms total duration.\n\tLast attempt: " + lastAttempt; + return cause == null ? new MaaSHttpException(message) : new MaaSHttpException(message, cause); + } - void afterStatus(int code, String body) { - count++; - lastFailure = null; - lastStatusAndBody = "status " + code + ", body: " + body; - } + /** A status the caller did not expect, but one worth another attempt. Never leaves this class. */ + private static final class RetryableStatus extends RuntimeException { + private final transient int code; + private final transient String body; - void afterTransportError(IOException e) { - count++; - lastFailure = e; - lastStatusAndBody = null; + RetryableStatus(int code, String body) { + super(null, null, false, false); + this.code = code; + this.body = body; } - String describeLast() { - if (lastStatusAndBody != null) { - return lastStatusAndBody; - } - return lastFailure != null ? lastFailure.toString() : "unknown"; + String describe() { + return "status " + code + ", body: " + body; } } - - /** - * The usual terminal failure: the backoff is clamped to the time left, so a call that keeps - * failing lands exactly on the deadline. Carries what the last attempt saw, otherwise the - * trace says only that a minute went by. - */ - private static MaaSHttpException totalDurationExceeded(Request req, Attempts attempts, long maxTotalMillis) { - String message = "Gave up on " + req + " after " + attempts.count + " retries: ran out of its " - + maxTotalMillis + "ms total duration." - + "\n\tLast attempt: " + attempts.describeLast(); - return attempts.lastFailure != null - ? new MaaSHttpException(message, attempts.lastFailure) - : new MaaSHttpException(message); - } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 685d094085..a7c10f8922 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -289,22 +289,6 @@ void testTotalDurationExceeded_CarriesTheLastFailureAsCause() throws IOException } } - // Delay must grow between attempts and saturate at a quarter of the total duration. - @Test - void testBackoffMillis_GrowsAndSaturatesAtTheCap() { - long[] expectedFor60s = {1_000, 2_000, 4_000, 8_000, 15_000, 15_000}; - for (int attempt = 1; attempt <= expectedFor60s.length; attempt++) { - assertEquals(expectedFor60s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 60_000), - "attempt " + attempt + " of a 60s total duration"); - } - - long[] expectedFor5s = {1_000, 1_250, 1_250}; - for (int attempt = 1; attempt <= expectedFor5s.length; attempt++) { - assertEquals(expectedFor5s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 5_000), - "attempt " + attempt + " of a 5s total duration"); - } - } - // A tight max total duration must cut the retry loop short well before the attempt count is exhausted. @Test void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServer) { diff --git a/maas-client/pom.xml b/maas-client/pom.xml index e74671c189..1c5652751e 100644 --- a/maas-client/pom.xml +++ b/maas-client/pom.xml @@ -24,6 +24,7 @@ 3.1.8 4.2.0 + 3.3.2 ${project.basedir}/../report-aggregate/target/site/jacoco-aggregate/jacoco.xml @@ -92,6 +93,11 @@ kafka-streams ${kafka.version} + + dev.failsafe + failsafe + ${failsafe.version} + From 5c940c4f67f3367b1648a774640ff2cad588a52e Mon Sep 17 00:00:00 2001 From: Ksiona Date: Mon, 10 Aug 2026 14:49:07 +0400 Subject: [PATCH 09/24] fix: retry behavior for maas-client, tests were added --- maas-client/CHANGELOG.md | 18 ++ maas-client/README.md | 41 ++++ .../cloud/maas/client/impl/Env.java | 19 ++ .../maas/client/impl/http/HttpExecution.java | 129 +++++++++++-- .../impl/kafka/KafkaMaaSClientImpl.java | 34 +++- .../impl/http/HttpExecutionFailoverTest.java | 178 ++++++++++++++++++ .../KafkaMaaSClientWatchBackoffTest.java | 116 ++++++++++++ .../impl/rabbit/RabbitFailoverTest.java | 110 +++++++++++ 8 files changed, 633 insertions(+), 12 deletions(-) create mode 100644 maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java create mode 100644 maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java create mode 100644 maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 7a1ca539a0..5213b9e9cb 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -1,5 +1,23 @@ # This page contains notably changes of maas-client project. +## Unreleased +* `Features` + - HTTP calls to maas-agent are now retried on retryable status codes, not only on `IOException`. + Retryable: 5xx, 429, **405** and **401**. See "Retry behaviour and configuration" in README for why + the two 4xx codes are included — without them the client does not survive a Postgres leader switchover. + - Backoff is exponential with jitter instead of a fixed 1s delay. + - New configuration: `maas.http.retry.max-total-duration-ms` (`60s` by default) — a single + setting bounding the whole call. The attempt count and the backoff growth are derived from + it, so there are no separate knobs to keep consistent. + - The Kafka topic `watch-create` long poll no longer goes through the retry policy + (`HttpExecution.noRetry()`); its own loop got a linear capped backoff instead, so a + down maas-agent is no longer polled in a hot loop. +* `Behaviour changes` + - **A call that fails with a retryable status now takes longer before failing.** Previously an + unexpected 5xx/405/401 threw immediately; it is now retried within the configured limits. + - Interrupting a thread during a retry wait now restores the interrupt flag and aborts the loop, + instead of swallowing `InterruptedException`. + ## 10.0.0 * `Features` - **Breaking:** Removed _MaaSAPIClient.loadConfiguration_ from public API. diff --git a/maas-client/README.md b/maas-client/README.md index 8839492463..4300040e1d 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -58,6 +58,47 @@ MaaSClient client = new MaaSAPIClientImpl(() -> M2MManager.getInstance().getToke ``` +## Retry behaviour and configuration + +Every call to maas-agent is retried before giving up, bounded by a single +setting: the maximum total duration of the call. + +| Property | Default | Meaning | +|---|---|---| +| `maas.http.timeout` | `30` (seconds) | connect/read/write timeout of a **single** attempt | +| `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included | + +`max-total-duration-ms` is the only retry knob. The number of attempts and the +growth of the pauses between them are derived from it, so there is nothing to +keep consistent by hand: the first pause is 1s, each next one doubles, and the +cap is a quarter of the total. With the default 60s that gives pauses of +1s, 2s, 4s, 8s, 15s, 15s — roughly six attempts before giving up. + +The default of 60s is chosen to outlast a database leader switchover, which is +the case these retries exist for, while still failing fast enough for a caller +to react to a real outage. + +Backoff is exponential with +/-20% jitter, so concurrent callers do not retry in +lockstep against a recovering agent. + +The watch endpoint (`watch-create`) is deliberately excluded: it is a long poll with +its own loop, so retrying inside the call would nest two policies and block the watch +for the whole duration. That loop has its own linear, capped backoff instead. + +Which responses are retried: + +| Response | Retried | Why | +|---|---|---| +| `IOException` | yes | connection refused/reset while the agent is being rescheduled | +| 5xx | yes | includes the `500` maas-agent returns when it cannot reach maas-service at all | +| 429 | yes | throttling | +| **405** | **yes** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a leader switchover arrives as `405`, not as `5xx` | +| **401** | **yes** | the M2M token is supplied per request, so an expired token or a briefly unavailable token provider clears itself on the next attempt | +| other 4xx | no | permanent client errors, failed on the first attempt | + +The two 4xx entries are deliberate. Applying the usual "retry 5xx, fail fast on +4xx" rule here means not surviving a database leader switchover. + ## Kafka client usage example All MaaS operations for Kafka is collected in [KafkaMaaSClient](https://github.com/Netcracker/qubership-maas-client/blob/main/client/src/main/java/com/netcracker/cloud/maas/client/api/kafka/KafkaMaaSClient.java). To obtain *new* instance of MaaS Kafka client just call: ```java diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index 4ef51aeb59..a99da48c2f 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -35,6 +35,7 @@ public class Env { public static final String PROP_TENANT_MANAGER_URL = "maas.client.tenant-manager.url"; public static final String PROP_TENANT_MANAGER_RECONNECT_TIMEOUT = "maas.client.tenant-manager.reconnect-timeout"; public static final String PROP_HTTP_TIMEOUT = "maas.http.timeout"; + public static final String PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS = "maas.http.retry.max-total-duration-ms"; public static String apiUrl() { return apiUrl(M2MClient.isK8sM2mEnabled()); @@ -109,6 +110,24 @@ public static Duration httpTimeout() { ); } + /** + * How long one call to maas-agent may take in total, retries included. This is the + * only retry knob: the number of attempts and the growth of the backoff are derived + * from it, so there is nothing to keep consistent by hand. + *

+ * The default of 60s is chosen to outlast a database leader switchover — the case the + * retries exist for — while still failing fast enough for a caller to react to a real + * outage. + */ + public static Duration httpRetryMaxTotalDuration() { + return Duration.ofMillis( + stringProperty(PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS) + .map(Long::parseLong) + .filter(ms -> ms > 0) + .orElse(60_000L) + ); + } + public static String url2ws(String url) { return url.replaceAll("^http(s?):", "ws$1:"); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 9b738d6838..a2faec2b01 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -4,12 +4,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.SneakyThrows; +import com.netcracker.cloud.maas.client.impl.Env; import lombok.extern.slf4j.Slf4j; import okhttp3.*; import java.io.IOException; import java.util.*; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -19,11 +21,11 @@ public class HttpExecution { public static final MediaType JSON = MediaType.parse("application/json; charset=utf-8"); - private final int RETRIES_NUMBER = 30; private final OkHttpClient httpClient; private final Request.Builder req; private final List expectedCodes = new ArrayList<>(); private final Map> errorHandler = new HashMap<>(); + private boolean retryEnabled = true; public static final ObjectMapper MAPPER = new ObjectMapper() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @@ -68,6 +70,14 @@ public HttpExecution supressError(int code, Consumer handler) { return this; } + /** + * Performs a single attempt and lets the caller decide what to do with a failure. + */ + public HttpExecution noRetry() { + this.retryEnabled = false; + return this; + } + private Function der(OmnivoreFunction deserializer) { return body -> { try { @@ -94,11 +104,96 @@ public Optional sendAndReceive(OmnivoreFunction responseDeseri return sendAndReceive().map(der(responseDeserializer)); } - @SneakyThrows + /** + * Two 4xx codes are deliberately retryable. Both are transient in this specific chain + * rather than permanent client errors: + *

    + *
  • 405 - maas-service maps PG error 25006 (READ ONLY SQL TRANSACTION) to + * {@code StatusMethodNotAllowed}, so a write against a demoted Patroni node + * during a leader switchover arrives here as 405, not as 5xx.
    • + *
  • 401 - the M2M token is supplied per request by the OkHttp interceptor, so an + * expired token or a briefly unavailable token provider resolves itself on the + * next attempt.
    • + *
+ */ + private static boolean isRetryableStatus(int code) { + if (code >= 500) { + return true; + } + return code == 429 || code == 405 || code == 401; + } + + /** First backoff pause. Not configurable*/ + private static final long BASE_DELAY_MILLIS = 1_000L; + + /** + * The cap on a single pause is derived from the total duration rather than configured + * separately. A quarter keeps the growth useful at both ends of the range: with the + * default 60s the pauses run 1s, 2s, 4s, 8s, 15s, 15s (~6 attempts), and with a 5s + * total they run 1s, 1.25s, 1.25s, 1.25s (~4 attempts). Either way the caller gets + * several tries without hammering an agent that is coming back up. + */ + private static final int MAX_DELAY_FRACTION_OF_TOTAL = 4; + + /** + * Delay before jitter: doubles per attempt, capped at the derived maximum. + * Doubling is done in integer arithmetic and saturates at the cap, so a large attempt + * count cannot overflow into a negative delay. + */ + static long cappedDelayMillis(int attempt) { + long max = Math.max(1L, Env.httpRetryMaxTotalDuration().toMillis() / MAX_DELAY_FRACTION_OF_TOTAL); + long delay = Math.min(BASE_DELAY_MILLIS, max); + for (int i = 1; i < attempt && delay < max; i++) { + delay = delay > max / 2 ? max : delay * 2; + } + return delay; + } + + // Exponential backoff with jitter between retries. + private static long backoffMillis(int attempt) { + long capped = cappedDelayMillis(attempt); + double jitterFactor = 0.8 + ThreadLocalRandom.current().nextDouble() * 0.4; + return Math.max(1L, (long) (capped * jitterFactor)); + } + + // The total duration is the only stop condition: there is no attempt counter to + // disagree with it. Callers that own a retry loop opt out entirely via noRetry(). + private boolean canRetry(long deadlineNanos) { + return retryEnabled && System.nanoTime() < deadlineNanos; + } + + /** + * Waits before the next retry. The delay is clamped to whatever is left of the max + * total duration, otherwise a backoff started just before the deadline would overshoot + * it by up to the configured max delay. Restores the interrupt flag and aborts if + * interrupted. + */ + private static void sleepBackoff(int attempt, long deadlineNanos) { + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + if (remainingMillis <= 0) { + return; + } + try { + Thread.sleep(Math.min(backoffMillis(attempt), remainingMillis)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting to retry maas-agent request", e); + } + } + + // OkHttp declares Response.body() as nullable; treat a missing body as empty rather + // than dereferencing it. Keeps throwing IOException so the caller's retry loop sees it. + private static String bodyAsString(Response response) throws IOException { + ResponseBody body = response.body(); + return body == null ? "" : body.string(); + } + private Optional sendAndReceive() { Request compiledReq = req.build(); log.debug("Send request: {}", compiledReq); + long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); + long deadlineNanos = System.nanoTime() + Env.httpRetryMaxTotalDuration().toNanos(); int attempt = 0; while (true) { try (Response response = httpClient.newCall(compiledReq).execute()) { @@ -106,24 +201,36 @@ private Optional sendAndReceive() { log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); if (errorHandler.containsKey(response.code())) { - errorHandler.get(response.code()).accept(response.body().string()); + errorHandler.get(response.code()).accept(bodyAsString(response)); return Optional.empty(); } if (!expectedCodes.contains(response.code())) { - throw new RuntimeException("Unexpected status code " + response.code() + " for request: " + compiledReq + "\n\tResponse body: " + response.body().string()); + if (isRetryableStatus(response.code()) && canRetry(deadlineNanos)) { + attempt++; + log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", + response.code(), compiledReq, attempt, maxTotalMillis); + sleepBackoff(attempt, deadlineNanos); + continue; + } + throw new RuntimeException("Unexpected status code " + response.code() + + " for request: " + compiledReq + + ", gave up after " + attempt + " retries" + + "\n\tResponse body: " + bodyAsString(response)); } - String body = response.body().string(); + String body = bodyAsString(response); log.debug("Response body: {}", body); return Optional.of(body); } catch (IOException e) { - if (attempt++ < RETRIES_NUMBER) { - log.warn("Error execute http request: {}, Retry {} of {}", e.getMessage(), attempt, RETRIES_NUMBER); - Thread.sleep(1000); - } else { - throw new RuntimeException("Error executing " + compiledReq + ". Number of " + RETRIES_NUMBER + " retries exceeded", e); + if (!canRetry(deadlineNanos)) { + throw new RuntimeException("Error executing " + compiledReq + + ", gave up after " + attempt + " retries", e); } + attempt++; + log.warn("Error execute http request: {}, Retry {}, within {}ms total", + e.getMessage(), attempt, maxTotalMillis); + sleepBackoff(attempt, deadlineNanos); } } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 4e9d0e62f8..6a7dc158ca 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -41,6 +41,8 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { private final ApiUrlProvider apiProvider; private final Duration watchTimeout = Duration.ofSeconds(60); + private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); + private static final Duration WATCH_MAX_RETRY_INTERVAL = Duration.ofSeconds(30); // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); private volatile boolean closed = false; @@ -110,6 +112,7 @@ public void watchTenantTopics(String name, Consumer> callback } private void watchTenantCreateTopics() { + int failures = 0; TypeReference> typeRef = new TypeReference<>() { }; while (!closed) { @@ -120,10 +123,21 @@ private void watchTenantCreateTopics() { found = httpClient.request(url) .post(topicCreateListeners.keySet()) .expect(200) + .noRetry() .sendAndReceive(typeRef) .orElse(Collections.emptyList()); + failures = 0; } catch (Exception e) { - log.error("Error execute request to {}", url, e); + // `closed` is the reliable stop signal: an interrupt can be swallowed by + // the HTTP/JSON layers before it reaches us, the flag cannot. + if (closed || Thread.currentThread().isInterrupted()) { + return; // shutting down, not a failure worth reporting + } + failures++; + log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); + if (!sleepWatchBackoff(failures)) { + return; // interrupted while backing off + } } for (TopicInfo addr : found) { @@ -160,6 +174,24 @@ private void watchTenantCreateTopics() { } } + /** + * Linear, capped backoff between failed watch polls, reset on every success. + * + * @return false if the thread was interrupted while waiting, meaning the caller should stop + */ + private static boolean sleepWatchBackoff(int failures) { + long delayMillis = Math.min( + failures * WATCH_RETRY_INTERVAL.toMillis(), + WATCH_MAX_RETRY_INTERVAL.toMillis()); + try { + Thread.sleep(delayMillis); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + @Override public void watchTopicCreate(String name, Consumer callback) { apiProvider.getServerApiVersion().requiresApiVersion(2, 8); diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java new file mode 100644 index 0000000000..2d9ded65f2 --- /dev/null +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -0,0 +1,178 @@ +package com.netcracker.cloud.maas.client.impl.http; + +import com.netcracker.cloud.maas.client.impl.Env; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.junit.jupiter.MockServerExtension; +import org.mockserver.matchers.Times; +import org.mockserver.verify.VerificationTimes; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +@ExtendWith(MockServerExtension.class) +class HttpExecutionFailoverTest { + + private static final String PATH = "/api/v1/kafka/topic"; + + @Test + void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(405) + .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertTrue(body.isPresent()); + assertTrue(body.get().equals("ok")); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(500) + .withBody("{\"error\":\"error proxying request: connection refused\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertTrue(body.isPresent()); + assertTrue(body.get().equals("ok")); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_401TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertTrue(body.isPresent()); + assertTrue(body.get().equals("ok")); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_400NotRetried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); + + withFastRetries(() -> + assertTrue(assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class) + ).getMessage().contains("400"))); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + + @Test + void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws Exception { + // A long total duration keeps the retry wait long enough for the interrupt to land in it. + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "600000", () -> { + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(Duration.ofMillis(300)) + .build(); + Request.Builder req = new Request.Builder().url("http://127.0.0.1:1/unreachable").get(); + HttpExecution execution = new HttpExecution(client, req); + execution.expect(200); + + AtomicBoolean interruptedAfter = new AtomicBoolean(); + AtomicReference thrown = new AtomicReference<>(); + CountDownLatch started = new CountDownLatch(1); + + Thread worker = new Thread(() -> { + started.countDown(); + try { + execution.sendAndReceive(String.class); + } catch (Throwable t) { + thrown.set(t); + } finally { + interruptedAfter.set(Thread.currentThread().isInterrupted()); + } + }, "http-execution-interrupt-test"); + worker.start(); + + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread.sleep(500); + worker.interrupt(); + worker.join(5000); + + assertFalse(worker.isAlive(), "worker should abort instead of continuing to retry after interrupt"); + assertTrue(interruptedAfter.get(), "interrupt flag must be restored after an interrupted retry wait"); + }); + } + + // Delay must grow between attempts, not stay flat at the base value. + @Test + void testBackoffMillis_GrowsBetweenAttempts() { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "60000", () -> { + long attempt1 = HttpExecution.cappedDelayMillis(1); + long attempt2 = HttpExecution.cappedDelayMillis(2); + long attempt3 = HttpExecution.cappedDelayMillis(3); + assertTrue(attempt2 > attempt1, "expected " + attempt2 + " > " + attempt1); + assertTrue(attempt3 > attempt2, "expected " + attempt3 + " > " + attempt2); + }); + } + + // A tight max total duration must cut the retry loop short well before the attempt count is exhausted. + @Test + void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "200", () -> { + long start = System.currentTimeMillis(); + assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class)); + long elapsedMs = System.currentTimeMillis() - start; + assertTrue(elapsedMs < 800, + "expected retry loop to abort near the 200ms max total duration, took " + elapsedMs + "ms"); + }); + } + + // A short total duration is now the only lever: it bounds both the number of attempts + // and the pauses between them (the cap is derived as a quarter of it). + private static void withFastRetries(Runnable test) { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + } + + private static HttpExecution execution(ClientAndServer mockServer) { + OkHttpClient client = new OkHttpClient(); + Request.Builder req = new Request.Builder() + .url("http://localhost:" + mockServer.getPort() + PATH) + .get(); + return new HttpExecution(client, req); + } +} diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java new file mode 100644 index 0000000000..a0faf57aeb --- /dev/null +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -0,0 +1,116 @@ +package com.netcracker.cloud.maas.client.impl.kafka; + +import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; +import com.netcracker.cloud.maas.client.impl.Env; +import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; +import com.netcracker.cloud.maas.client.impl.http.HttpClient; +import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +/** + * Pins the invariant that a failing {@code watch-create} long poll is retried with a backoff + * rather than in a hot loop. + * + *

Without one, a maas-agent that is down gets polled as fast as the socket can refuse the + * connection — hammering it exactly while it is coming back up. + */ +class KafkaMaaSClientWatchBackoffTest { + + private static final String WATCHED_TOPIC = "orders"; + private static final String NAMESPACE = "cloud-dev"; + + /** + * The backoff is linear at one second per consecutive failure, so this window admits the + * first poll, a 1s pause, the second poll and a 2s pause. Anything much above that means + * the loop is not backing off at all. + */ + private static final long OBSERVATION_WINDOW_MILLIS = 2_500; + private static final int MAX_EXPECTED_POLLS = 5; + + private final AtomicInteger watchPolls = new AtomicInteger(); + private final CountDownLatch firstPoll = new CountDownLatch(1); + private HttpServer agentStub; + private KafkaMaaSClientImpl client; + + @BeforeEach + void startAgentStub() throws IOException { + agentStub = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + agentStub.createContext("/api-version", exchange -> respond(exchange, 200, "{\"major\": 2, \"minor\": 8}")); + agentStub.createContext("/api/v2/kafka/topic/watch-create", this::failWatchPoll); + agentStub.start(); + } + + @AfterEach + void stopClientAndStub() { + if (client != null) { + client.close(); + } + agentStub.stop(0); + } + + @Test + void failingWatchPollIsBackedOffInsteadOfHotLooping() { + withProp(Env.PROP_NAMESPACE, NAMESPACE, () -> { + String agentUrl = "http://localhost:" + agentStub.getAddress().getPort(); + withProp(Env.PROP_MAAS_AGENT_URL, agentUrl, () -> { + client = createKafkaClient(agentUrl); + client.watchTopicCreate(WATCHED_TOPIC, addr -> { /* never created in this test */ }); + + assertTrue(firstPoll.await(10, TimeUnit.SECONDS), + "the watch thread never reached the agent stub, so nothing was measured"); + Thread.sleep(OBSERVATION_WINDOW_MILLIS); + + int polls = watchPolls.get(); + // The lower bound matters as much as the upper one: without it the assertion + // would also pass when the loop never ran and nothing was verified. + assertTrue(polls >= 1, "watch loop did not poll at all, the test would pass vacuously"); + assertTrue(polls <= MAX_EXPECTED_POLLS, + "expected the watch loop to back off between failures, but it polled " + polls + + " times in " + OBSERVATION_WINDOW_MILLIS + "ms (limit " + MAX_EXPECTED_POLLS + ")"); + }); + }); + } + + private static KafkaMaaSClientImpl createKafkaClient(String agentUrl) { + System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, agentUrl); + var httpClient = HttpClient.getMaasClient(() -> "faketoken"); + var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); + System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); + + return new KafkaMaaSClientImpl(httpClient, null, new ApiUrlProvider(serverApiVersion, agentUrl)); + } + + /** Answers every poll with 500, the code maas-agent returns when it cannot reach maas-service. */ + private void failWatchPoll(HttpExchange exchange) throws IOException { + watchPolls.incrementAndGet(); + firstPoll.countDown(); + respond(exchange, 500, "{\"error\":\"error proxying request: maas-service unavailable\"}"); + } + + private static void respond(HttpExchange exchange, int status, String body) throws IOException { + exchange.getRequestBody().readAllBytes(); + + byte[] payload = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(status, payload.length); + try (OutputStream response = exchange.getResponseBody()) { + response.write(payload); + } + } +} diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java new file mode 100644 index 0000000000..26e8490fda --- /dev/null +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -0,0 +1,110 @@ +package com.netcracker.cloud.maas.client.impl.rabbit; + +import com.netcracker.cloud.maas.client.api.Classifier; +import com.netcracker.cloud.maas.client.api.rabbit.VHost; +import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; +import com.netcracker.cloud.maas.client.impl.Env; +import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; +import com.netcracker.cloud.maas.client.impl.http.HttpClient; +import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockserver.integration.ClientAndServer; +import org.mockserver.junit.jupiter.MockServerExtension; +import org.mockserver.matchers.Times; +import org.mockserver.verify.VerificationTimes; + +import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +@ExtendWith(MockServerExtension.class) +class RabbitFailoverTest { + + private static final String PATH = "/api/v2/rabbit/vhost"; + + @BeforeEach + void reset(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath("/api-version")) + .respond(response().withBody("{\"major\":2, \"minor\": 16}")); + } + + @Test + void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(405) + .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}")); + mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody(""" + { + "cnn": "ampq://rabbit-cluster:4321/maas.core-dev.123456", + "username": "testuser", + "password": "plain:testpassword" + } + """)); + + withProp(Env.PROP_NAMESPACE, "core-dev", () -> + withFastRetries(() -> { + RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); + VHost vhost = client.getOrCreateVirtualHost(new Classifier("commands")); + assertNotNull(vhost); + })); + + mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { + mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) + .respond(response().withStatusCode(500) + .withBody("{\"error\":\"error proxying request: connection refused\"}")); + mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody(""" + { + "cnn": "ampq://rabbit-cluster:4321/maas.core-dev.123456", + "username": "testuser", + "password": "plain:testpassword" + } + """)); + + withProp(Env.PROP_NAMESPACE, "core-dev", () -> + withFastRetries(() -> { + RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); + VHost vhost = client.getOrCreateVirtualHost(new Classifier("commands")); + assertNotNull(vhost); + })); + + mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(3)); + } + + @Test + void testFailover_400NotRetried(ClientAndServer mockServer) { + mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); + + withProp(Env.PROP_NAMESPACE, "core-dev", () -> + withFastRetries(() -> { + RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); + org.junit.jupiter.api.Assertions.assertThrows(RuntimeException.class, + () -> client.getOrCreateVirtualHost(new Classifier("commands"))); + })); + + mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(1)); + } + + // A short total duration is now the only lever: it bounds both the number of attempts + // and the pauses between them (the cap is derived as a quarter of it). + private static void withFastRetries(Runnable test) { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + } + + private static RabbitMaaSClientImpl createRabbitClient(String agentUrl) { + System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, agentUrl); + var httpClient = HttpClient.getMaasClient(() -> "faketoken"); + var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); + return new RabbitMaaSClientImpl(httpClient, new ApiUrlProvider(serverApiVersion, agentUrl)); + } +} From 13ffc81b5d25724feaadae9f2f53c4cb90480fed Mon Sep 17 00:00:00 2001 From: Ksiona Date: Mon, 10 Aug 2026 19:48:03 +0400 Subject: [PATCH 10/24] fix: sonar on old code + retry rule for 405 rc --- maas-client/CHANGELOG.md | 11 +- maas-client/README.md | 34 ++-- .../cloud/maas/client/impl/Env.java | 10 +- .../maas/client/impl/http/HttpExecution.java | 168 ++++++++++++------ .../impl/kafka/KafkaMaaSClientImpl.java | 45 ++++- .../impl/http/HttpExecutionFailoverTest.java | 131 ++++++++++++-- .../impl/rabbit/RabbitFailoverTest.java | 17 +- 7 files changed, 314 insertions(+), 102 deletions(-) diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 5213b9e9cb..cdb1907245 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -3,12 +3,15 @@ ## Unreleased * `Features` - HTTP calls to maas-agent are now retried on retryable status codes, not only on `IOException`. - Retryable: 5xx, 429, **405** and **401**. See "Retry behaviour and configuration" in README for why - the two 4xx codes are included — without them the client does not survive a Postgres leader switchover. + Retryable: 5xx, 429, **405** (only when the body carries a maas-service error) and **401** + (once). See "Retry behaviour and configuration" in README for why the two 4xx codes are + included — without them the client does not survive a Postgres leader switchover. - Backoff is exponential with jitter instead of a fixed 1s delay. - New configuration: `maas.http.retry.max-total-duration-ms` (`60s` by default) — a single setting bounding the whole call. The attempt count and the backoff growth are derived from - it, so there are no separate knobs to keep consistent. + it, so there are no separate knobs to keep consistent. Every attempt is bounded by what is + left of it, so with the defaults the worst case a caller can see is ~60s, not 60s plus one + `maas.http.timeout`. - The Kafka topic `watch-create` long poll no longer goes through the retry policy (`HttpExecution.noRetry()`); its own loop got a linear capped backoff instead, so a down maas-agent is no longer polled in a hot loop. @@ -17,6 +20,8 @@ unexpected 5xx/405/401 threw immediately; it is now retried within the configured limits. - Interrupting a thread during a retry wait now restores the interrupt flag and aborts the loop, instead of swallowing `InterruptedException`. + - `KafkaMaaSClient.watchTopicCreate` throws `IllegalStateException` after `close()`, instead of + registering a callback that can never fire. ## 10.0.0 * `Features` diff --git a/maas-client/README.md b/maas-client/README.md index 4300040e1d..c1df3f53cd 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -68,22 +68,22 @@ setting: the maximum total duration of the call. | `maas.http.timeout` | `30` (seconds) | connect/read/write timeout of a **single** attempt | | `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included | -`max-total-duration-ms` is the only retry knob. The number of attempts and the -growth of the pauses between them are derived from it, so there is nothing to -keep consistent by hand: the first pause is 1s, each next one doubles, and the -cap is a quarter of the total. With the default 60s that gives pauses of -1s, 2s, 4s, 8s, 15s, 15s — roughly six attempts before giving up. +`max-total-duration-ms` is the only retry knob: the attempt count and the pauses +between attempts are derived from it. The first pause is 1s, each next one +doubles, and the cap is a quarter of the total — with the default 60s that gives +1s, 2s, 4s, 8s, 15s, 15s, roughly six attempts when each attempt fails fast. If +attempts hang instead, fewer of them fit into the same budget. Backoff carries ++/-20% jitter so concurrent callers do not retry in lockstep. -The default of 60s is chosen to outlast a database leader switchover, which is -the case these retries exist for, while still failing fast enough for a caller -to react to a real outage. +Each attempt is additionally bounded by what is left of the total duration, so +the worst case a caller sees is the budget itself rather than the budget plus one +`maas.http.timeout`. -Backoff is exponential with +/-20% jitter, so concurrent callers do not retry in -lockstep against a recovering agent. +The 60s default is meant to outlast a database leader switchover while still +failing fast enough to react to a real outage. -The watch endpoint (`watch-create`) is deliberately excluded: it is a long poll with -its own loop, so retrying inside the call would nest two policies and block the watch -for the whole duration. That loop has its own linear, capped backoff instead. +The watch endpoint (`watch-create`) is excluded: it is a long poll with its own +loop and its own backoff. Which responses are retried: @@ -92,12 +92,12 @@ Which responses are retried: | `IOException` | yes | connection refused/reset while the agent is being rescheduled | | 5xx | yes | includes the `500` maas-agent returns when it cannot reach maas-service at all | | 429 | yes | throttling | -| **405** | **yes** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a leader switchover arrives as `405`, not as `5xx` | -| **401** | **yes** | the M2M token is supplied per request, so an expired token or a briefly unavailable token provider clears itself on the next attempt | +| **405** | **only with a maas-service error body** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a switchover arrives as `405`, not as `5xx`. A plain `405` — a route removed on the server, an ingress rejecting the method — is permanent and fails fast | +| **401** | **once** | covers a token that expired in flight. Further attempts re-send the same token, since the supplier cannot be told it was rejected | | other 4xx | no | permanent client errors, failed on the first attempt | -The two 4xx entries are deliberate. Applying the usual "retry 5xx, fail fast on -4xx" rule here means not surviving a database leader switchover. +The two 4xx entries are deliberate: the usual "retry 5xx, fail fast on 4xx" rule +does not survive a database leader switchover here. ## Kafka client usage example All MaaS operations for Kafka is collected in [KafkaMaaSClient](https://github.com/Netcracker/qubership-maas-client/blob/main/client/src/main/java/com/netcracker/cloud/maas/client/api/kafka/KafkaMaaSClient.java). To obtain *new* instance of MaaS Kafka client just call: diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index a99da48c2f..8149d05aa8 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -111,13 +111,9 @@ public static Duration httpTimeout() { } /** - * How long one call to maas-agent may take in total, retries included. This is the - * only retry knob: the number of attempts and the growth of the backoff are derived - * from it, so there is nothing to keep consistent by hand. - *

- * The default of 60s is chosen to outlast a database leader switchover — the case the - * retries exist for — while still failing fast enough for a caller to react to a real - * outage. + * How long one call to maas-agent may take in total, retries included. The only retry + * knob: attempt count and backoff growth are derived from it. The 60s default outlasts + * a database leader switchover. */ public static Duration httpRetryMaxTotalDuration() { return Duration.ofMillis( diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index a2faec2b01..566a64d828 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -9,6 +9,7 @@ import okhttp3.*; import java.io.IOException; +import java.time.Duration; import java.util.*; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @@ -70,9 +71,7 @@ public HttpExecution supressError(int code, Consumer handler) { return this; } - /** - * Performs a single attempt and lets the caller decide what to do with a failure. - */ + /** Performs a single attempt. For callers that own a retry loop, such as a long poll. */ public HttpExecution noRetry() { this.retryEnabled = false; return this; @@ -104,44 +103,70 @@ public Optional sendAndReceive(OmnivoreFunction responseDeseri return sendAndReceive().map(der(responseDeserializer)); } + /** Code carried by every maas-service TMF error envelope. */ + private static final String MAAS_ERROR_CODE = "MAAS-0600"; + /** - * Two 4xx codes are deliberately retryable. Both are transient in this specific chain - * rather than permanent client errors: - *

    - *
  • 405 - maas-service maps PG error 25006 (READ ONLY SQL TRANSACTION) to - * {@code StatusMethodNotAllowed}, so a write against a demoted Patroni node - * during a leader switchover arrives here as 405, not as 5xx.
    • - *
  • 401 - the M2M token is supplied per request by the OkHttp interceptor, so an - * expired token or a briefly unavailable token provider resolves itself on the - * next attempt.
    • - *
+ * Two 4xx are transient here rather than permanent: 405 is how maas-service reports a + * read-only Postgres during a leader switchover, and 401 clears when the token is + * re-supplied on the next attempt. + *

+ * The 405 case is gated on the response body, because a plain 405 — a route removed on + * the server, an ingress rejecting the method — is permanent and must fail fast. */ - private static boolean isRetryableStatus(int code) { + private static boolean isRetryableStatus(int code, String body) { if (code >= 500) { return true; } - return code == 429 || code == 405 || code == 401; + if (code == 429 || code == 401) { + return true; + } + return code == 405 && isDatabaseUnavailable(body); } - /** First backoff pause. Not configurable*/ - private static final long BASE_DELAY_MILLIS = 1_000L; + /** + * Recognises the 405 that maas-service returns for PostgreSQL error 25006, mapped from + * {@code DatabaseIsReadonlyError} / {@code DatabaseIsNotActiveError}. Matched on the + * reason text because the TMF code is the same for every maas-service error. + */ + private static boolean isDatabaseUnavailable(String body) { + if (body == null || !body.contains(MAAS_ERROR_CODE)) { + return false; + } + return body.contains("read-only") || body.contains("not in 'active' mode"); + } /** - * The cap on a single pause is derived from the total duration rather than configured - * separately. A quarter keeps the growth useful at both ends of the range: with the - * default 60s the pauses run 1s, 2s, 4s, 8s, 15s, 15s (~6 attempts), and with a 5s - * total they run 1s, 1.25s, 1.25s, 1.25s (~4 attempts). Either way the caller gets - * several tries without hammering an agent that is coming back up. + * How many times a single call retries a 401. One is enough: it covers a token that + * expired in flight. A token the supplier still considers valid but the server rejects + * comes back identical on every further attempt. */ + static final int MAX_AUTH_RETRIES = 1; + + private int authAttempts = 0; + + private boolean canRetryStatus(int code, String body, long deadlineNanos) { + if (!isRetryableStatus(code, body) || !canRetry(deadlineNanos)) { + return false; + } + if (code == 401) { + return ++authAttempts <= MAX_AUTH_RETRIES; + } + return true; + } + + /** First backoff pause. */ + private static final long BASE_DELAY_MILLIS = 1_000L; + + /** A single pause is capped at this fraction of the total duration. */ private static final int MAX_DELAY_FRACTION_OF_TOTAL = 4; /** - * Delay before jitter: doubles per attempt, capped at the derived maximum. - * Doubling is done in integer arithmetic and saturates at the cap, so a large attempt - * count cannot overflow into a negative delay. + * Delay before jitter: doubles per attempt, capped. Integer arithmetic saturating at + * the cap, so a large attempt count cannot overflow. */ - static long cappedDelayMillis(int attempt) { - long max = Math.max(1L, Env.httpRetryMaxTotalDuration().toMillis() / MAX_DELAY_FRACTION_OF_TOTAL); + static long cappedDelayMillis(int attempt, long maxTotalMillis) { + long max = Math.max(1L, maxTotalMillis / MAX_DELAY_FRACTION_OF_TOTAL); long delay = Math.min(BASE_DELAY_MILLIS, max); for (int i = 1; i < attempt && delay < max; i++) { delay = delay > max / 2 ? max : delay * 2; @@ -150,53 +175,94 @@ static long cappedDelayMillis(int attempt) { } // Exponential backoff with jitter between retries. - private static long backoffMillis(int attempt) { - long capped = cappedDelayMillis(attempt); + private static long backoffMillis(int attempt, long maxTotalMillis) { + long capped = cappedDelayMillis(attempt, maxTotalMillis); double jitterFactor = 0.8 + ThreadLocalRandom.current().nextDouble() * 0.4; return Math.max(1L, (long) (capped * jitterFactor)); } - // The total duration is the only stop condition: there is no attempt counter to - // disagree with it. Callers that own a retry loop opt out entirely via noRetry(). + // The total duration is the only stop condition, unless noRetry() was used. private boolean canRetry(long deadlineNanos) { return retryEnabled && System.nanoTime() < deadlineNanos; } /** - * Waits before the next retry. The delay is clamped to whatever is left of the max - * total duration, otherwise a backoff started just before the deadline would overshoot - * it by up to the configured max delay. Restores the interrupt flag and aborts if - * interrupted. + * Waits before the next retry, clamped to what is left of the total duration so the + * backoff cannot overshoot it. Restores the interrupt flag and aborts if interrupted. */ - private static void sleepBackoff(int attempt, long deadlineNanos) { - long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - if (remainingMillis <= 0) { + private static void sleepBackoff(int attempt, long maxTotalMillis, long deadlineNanos) { + long remaining = remainingMillis(deadlineNanos); + if (remaining <= 0) { return; } try { - Thread.sleep(Math.min(backoffMillis(attempt), remainingMillis)); + Thread.sleep(Math.min(backoffMillis(attempt, maxTotalMillis), remaining)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted while waiting to retry maas-agent request", e); } } - // OkHttp declares Response.body() as nullable; treat a missing body as empty rather - // than dereferencing it. Keeps throwing IOException so the caller's retry loop sees it. + // Response.body() is nullable in OkHttp; a missing body reads as empty. private static String bodyAsString(Response response) throws IOException { ResponseBody body = response.body(); return body == null ? "" : body.string(); } + /** + * Body of a non-2xx response, for the retry decision and the error message. Never throws: + * a body that cannot be read must not turn a permanent status into a retry. + */ + private static String errorBodyOrPlaceholder(Response response) { + try { + return bodyAsString(response); + } catch (IOException e) { + log.debug("Could not read error response body", e); + return ""; + } + } + + private static long remainingMillis(long deadlineNanos) { + return TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + } + + private static String giveUpSuffix(int attempt) { + return attempt == 0 ? "" : ", gave up after " + attempt + " retries"; + } + + /** + * Client for one attempt, bounded by what is left of the total duration. Without it an + * attempt starting just before the deadline still runs for the full + * {@code maas.http.timeout} and the call overruns its budget. + *

+ * Not applied under {@link #noRetry()}: there the caller owns the lifecycle, and the + * watch long poll legitimately runs as long as the budget itself. + */ + private OkHttpClient clientForAttempt(long remainingMs) { + if (!retryEnabled) { + return httpClient; + } + // newBuilder shares the connection pool and dispatcher, so this is cheap + return httpClient.newBuilder() + .callTimeout(Duration.ofMillis(remainingMs)) + .build(); + } + private Optional sendAndReceive() { Request compiledReq = req.build(); log.debug("Send request: {}", compiledReq); long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); - long deadlineNanos = System.nanoTime() + Env.httpRetryMaxTotalDuration().toNanos(); + long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); int attempt = 0; while (true) { - try (Response response = httpClient.newCall(compiledReq).execute()) { + long remainingMs = remainingMillis(deadlineNanos); + if (remainingMs <= 0) { + throw new RuntimeException("Gave up on " + compiledReq + " after " + attempt + + " retries: the " + maxTotalMillis + "ms budget is spent"); + } + + try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { // check response codes against acceptable list log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); @@ -206,17 +272,20 @@ private Optional sendAndReceive() { } if (!expectedCodes.contains(response.code())) { - if (isRetryableStatus(response.code()) && canRetry(deadlineNanos)) { + // read once, without throwing: a body that cannot be read must not turn a + // permanent status into a retry + String errorBody = errorBodyOrPlaceholder(response); + if (canRetryStatus(response.code(), errorBody, deadlineNanos)) { attempt++; log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", response.code(), compiledReq, attempt, maxTotalMillis); - sleepBackoff(attempt, deadlineNanos); + sleepBackoff(attempt, maxTotalMillis, deadlineNanos); continue; } throw new RuntimeException("Unexpected status code " + response.code() + " for request: " + compiledReq - + ", gave up after " + attempt + " retries" - + "\n\tResponse body: " + bodyAsString(response)); + + giveUpSuffix(attempt) + + "\n\tResponse body: " + errorBody); } String body = bodyAsString(response); @@ -224,13 +293,12 @@ private Optional sendAndReceive() { return Optional.of(body); } catch (IOException e) { if (!canRetry(deadlineNanos)) { - throw new RuntimeException("Error executing " + compiledReq - + ", gave up after " + attempt + " retries", e); + throw new RuntimeException("Error executing " + compiledReq + giveUpSuffix(attempt), e); } attempt++; log.warn("Error execute http request: {}, Retry {}, within {}ms total", e.getMessage(), attempt, maxTotalMillis); - sleepBackoff(attempt, deadlineNanos); + sleepBackoff(attempt, maxTotalMillis, deadlineNanos); } } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 6a7dc158ca..e50badf25a 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -46,6 +46,14 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); private volatile boolean closed = false; + /** + * Monitor for parking the watch thread while there is nothing to watch. + *

+ * Not the thread itself: {@link Thread#join()} waits on that same monitor and the JVM + * notifies it when the thread ends, so a notification meant for the watch loop can be + * consumed by a {@code join()} in {@link #close()} and the loop never wakes up. + */ + private final Object watchLock = new Object(); private final Lazy watchThread = new Lazy<>(() -> { Thread exec = new Thread(this::watchTenantCreateTopics, "watchTopicCreate"); exec.setDaemon(true); @@ -92,7 +100,11 @@ public boolean deleteTopic(Classifier classifier) { .sendAndReceive(TopicDeleteResponse.class) .orElse(null); - if (resp != null && !resp.getFailedToDelete().isEmpty()) { + if (resp == null) { + // empty body: nothing was reported as deleted + return false; + } + if (!resp.getFailedToDelete().isEmpty()) { throw new MaaSException("Error delete topic by classifier: %s. Error: %s", classifier, resp.getFailedToDelete().get(0).getMessage()); } @@ -128,16 +140,20 @@ private void watchTenantCreateTopics() { .orElse(Collections.emptyList()); failures = 0; } catch (Exception e) { - // `closed` is the reliable stop signal: an interrupt can be swallowed by - // the HTTP/JSON layers before it reaches us, the flag cannot. - if (closed || Thread.currentThread().isInterrupted()) { + // `closed` is checked too: an interrupt can be swallowed further down + if (closed) { return; // shutting down, not a failure worth reporting } + if (Thread.currentThread().isInterrupted()) { + log.warn("Watch thread interrupted without close(), stopping to watch {}", url, e); + return; + } failures++; log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); if (!sleepWatchBackoff(failures)) { return; // interrupted while backing off } + continue; // `found` is still empty, nothing to deliver } for (TopicInfo addr : found) { @@ -164,11 +180,15 @@ private void watchTenantCreateTopics() { try { log.info("Nothing to watch, sleep thread."); - synchronized (watchThread.get()) { - watchThread.get().wait(); + synchronized (watchLock) { + // guarded wait: a bare wait() would also return on a spurious wakeup + while (!closed && topicCreateListeners.isEmpty()) { + watchLock.wait(); + } } log.info("Woke up!"); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); return; // exit loop } } @@ -194,12 +214,18 @@ private static boolean sleepWatchBackoff(int failures) { @Override public void watchTopicCreate(String name, Consumer callback) { + if (closed) { + // the watch thread has already exited and nothing restarts it, so the callback + // would never fire + throw new IllegalStateException("Client is closed, cannot watch topic: " + name); + } apiProvider.getServerApiVersion().requiresApiVersion(2, 8); log.info("Add watch for topic by: {}, callback: {}", name, callback); topicCreateListeners.computeIfAbsent(new Classifier(name), k -> Collections.synchronizedList(new ArrayList<>())).add(callback); - synchronized (watchThread.get()) { - watchThread.get().notify(); + watchThread.get(); // start the thread if this is the first watch + synchronized (watchLock) { + watchLock.notifyAll(); } } @@ -261,6 +287,9 @@ public List search(SearchCriteria criteria) { @Override public void close() { closed = true; + synchronized (watchLock) { + watchLock.notifyAll(); // release the watch thread if it is parked + } if (watchThread.isInitialized()) { watchThread.get().interrupt(); try { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 2d9ded65f2..51d9d6530c 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -10,7 +10,13 @@ import org.mockserver.matchers.Times; import org.mockserver.verify.VerificationTimes; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -18,6 +24,7 @@ import java.util.concurrent.atomic.AtomicReference; import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -41,7 +48,7 @@ void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { withFastRetries(() -> { Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); assertTrue(body.isPresent()); - assertTrue(body.get().equals("ok")); + assertEquals("ok", body.get()); }); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); @@ -59,16 +66,17 @@ void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { withFastRetries(() -> { Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); assertTrue(body.isPresent()); - assertTrue(body.get().equals("ok")); + assertEquals("ok", body.get()); }); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); } + /** An expired token clears on the next attempt, because the supplier is called again. */ @Test - void testFailover_401TwiceThenSuccess(ClientAndServer mockServer) { + void testFailover_401ThenSuccess(ClientAndServer mockServer) { mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.exactly(2)) + mockServer.when(request().withPath(PATH), Times.exactly(1)) .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(200).withBody("\"ok\"")); @@ -76,10 +84,97 @@ void testFailover_401TwiceThenSuccess(ClientAndServer mockServer) { withFastRetries(() -> { Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); assertTrue(body.isPresent()); - assertTrue(body.get().equals("ok")); + assertEquals("ok", body.get()); }); - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(2)); + } + + /** + * A 401 that keeps coming back means the supplier is handing out a token the server + * rejects, and it has no way of being told so. Further attempts resend the same token, + * so the budget is deliberately tighter than the overall duration: a wrong secret must + * fail fast instead of hanging for the whole minute. + */ + @Test + void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); + + withFastRetries(() -> + assertTrue(assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class) + ).getMessage().contains("401"))); + + mockServer.verify(request().withPath(PATH), + VerificationTimes.exactly(HttpExecution.MAX_AUTH_RETRIES + 1)); + } + + /** + * A 405 without a maas-service error envelope is an ordinary "method not allowed" — + * a route or an ingress rejecting the request — and must fail fast. + */ + @Test + void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(405).withBody("Method Not Allowed")); + + withFastRetries(() -> + assertTrue(assertThrows(RuntimeException.class, + () -> execution(mockServer).expect(200).sendAndReceive(String.class) + ).getMessage().contains("405"))); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + + /** + * The per-attempt budget is clamped to what is left of the total duration, so a hanging + * agent cannot stretch the call past it. + */ + @Test + void testMaxTotalDuration_BoundsAHangingAttempt() throws Exception { + // accepts the connection and never answers, unlike a refused connect which fails fast + try (ServerSocket silentServer = new ServerSocket(0)) { + List accepted = Collections.synchronizedList(new ArrayList<>()); + Thread acceptor = new Thread(() -> { + try { + while (!silentServer.isClosed()) { + accepted.add(silentServer.accept()); + } + } catch (IOException e) { + // the socket was closed, the test is over + } + }, "silent-server"); + acceptor.setDaemon(true); + acceptor.start(); + + try { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1000", () -> { + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(Duration.ofMinutes(1)) + .build(); + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + silentServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(client, req).expect(200); + + long start = System.currentTimeMillis(); + assertThrows(RuntimeException.class, () -> execution.sendAndReceive(String.class)); + long elapsedMs = System.currentTimeMillis() - start; + assertTrue(elapsedMs < 20_000, + "expected the call to be bounded by its 1000ms budget rather than by the " + + "one minute read timeout, took " + elapsedMs + "ms"); + }); + } finally { + synchronized (accepted) { + for (Socket socket : accepted) { + socket.close(); + } + } + } + } } @Test @@ -133,16 +228,20 @@ void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws Exception { }); } - // Delay must grow between attempts, not stay flat at the base value. + // Delay must grow between attempts and saturate at a quarter of the total duration. @Test - void testBackoffMillis_GrowsBetweenAttempts() { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "60000", () -> { - long attempt1 = HttpExecution.cappedDelayMillis(1); - long attempt2 = HttpExecution.cappedDelayMillis(2); - long attempt3 = HttpExecution.cappedDelayMillis(3); - assertTrue(attempt2 > attempt1, "expected " + attempt2 + " > " + attempt1); - assertTrue(attempt3 > attempt2, "expected " + attempt3 + " > " + attempt2); - }); + void testBackoffMillis_GrowsAndSaturatesAtTheCap() { + long[] expectedFor60s = {1_000, 2_000, 4_000, 8_000, 15_000, 15_000}; + for (int attempt = 1; attempt <= expectedFor60s.length; attempt++) { + assertEquals(expectedFor60s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 60_000), + "attempt " + attempt + " of a 60s budget"); + } + + long[] expectedFor5s = {1_000, 1_250, 1_250}; + for (int attempt = 1; attempt <= expectedFor5s.length; attempt++) { + assertEquals(expectedFor5s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 5_000), + "attempt " + attempt + " of a 5s budget"); + } } // A tight max total duration must cut the retry loop short well before the attempt count is exhausted. @@ -165,7 +264,7 @@ void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServ // A short total duration is now the only lever: it bounds both the number of attempts // and the pauses between them (the cap is derived as a quarter of it). private static void withFastRetries(Runnable test) { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "5000", test::run); } private static HttpExecution execution(ClientAndServer mockServer) { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java index 26e8490fda..a374ca1ddf 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -7,6 +7,7 @@ import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; import com.netcracker.cloud.maas.client.impl.http.HttpClient; import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -25,13 +26,27 @@ class RabbitFailoverTest { private static final String PATH = "/api/v2/rabbit/vhost"; + private String savedAgentUrl; + @BeforeEach void reset(ClientAndServer mockServer) { + savedAgentUrl = System.getProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); mockServer.reset(); mockServer.when(request().withPath("/api-version")) .respond(response().withBody("{\"major\":2, \"minor\": 16}")); } + // the agent url points at a mock server port that is gone once this class is done, + // so it must not leak into the rest of the JVM + @AfterEach + void restoreAgentUrl() { + if (savedAgentUrl == null) { + System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); + } else { + System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, savedAgentUrl); + } + } + @Test void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) @@ -98,7 +113,7 @@ void testFailover_400NotRetried(ClientAndServer mockServer) { // A short total duration is now the only lever: it bounds both the number of attempts // and the pauses between them (the cap is derived as a quarter of it). private static void withFastRetries(Runnable test) { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "2000", test::run); + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "5000", test::run); } private static RabbitMaaSClientImpl createRabbitClient(String agentUrl) { From 86852e93292f80f6fc823e93d160e20d1f77282d Mon Sep 17 00:00:00 2001 From: Ksiona Date: Mon, 10 Aug 2026 22:52:31 +0400 Subject: [PATCH 11/24] fix: sonar issues --- .../cloud/bluegreen/AbstractBGTest.java | 28 +++ maas-client/CHANGELOG.md | 2 + .../cloud/maas/client/api/MaaSException.java | 5 + .../maas/client/api/MaaSHttpException.java | 17 ++ .../cloud/maas/client/impl/Env.java | 7 +- .../maas/client/impl/http/HttpExecution.java | 9 +- .../impl/kafka/KafkaMaaSClientImpl.java | 100 ++++++++- .../impl/http/HttpExecutionFailoverTest.java | 194 ++++++++++-------- .../KafkaMaaSClientWatchBackoffTest.java | 48 ++--- .../impl/rabbit/RabbitFailoverTest.java | 6 +- 10 files changed, 286 insertions(+), 130 deletions(-) create mode 100644 maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java index 495c218c5a..c3be442445 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java @@ -8,6 +8,7 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -30,6 +31,8 @@ class AbstractBGTest { String consulUrl; + private static final Duration NODE_REGISTRATION_TIMEOUT = Duration.ofSeconds(30); + @Container ConsulContainer consulContainer = new ConsulContainer("hashicorp/consul:1.16") .waitingFor(Wait.forHttp("/v1/catalog/nodes") @@ -39,6 +42,31 @@ class AbstractBGTest { @BeforeEach void before() { consulUrl = String.format("http://%s:%d", consulContainer.getHost(), consulContainer.getMappedPort(8500)); + awaitNodeRegistered(); + } + + /** + * The agent answers on its port before it has registered itself in the catalog, and a session + * cannot be bound to a node that is not there yet: consul replies 500 "Missing node registration". + */ + private void awaitNodeRegistered() { + Instant deadline = Instant.now().plus(NODE_REGISTRATION_TIMEOUT); + String lastSeen = "no response"; + while (Instant.now().isBefore(deadline)) { + try { + String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), + String.class).sendAndGet(); + lastSeen = nodes; + if (nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]")) { + return; + } + } catch (Exception e) { + lastSeen = e.toString(); + } + run(() -> Thread.sleep(100)); + } + throw new IllegalStateException("Consul node was not registered in the catalog within " + + NODE_REGISTRATION_TIMEOUT + ", last response: " + lastSeen); } @SneakyThrows diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index cdb1907245..4ba81264e4 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -22,6 +22,8 @@ instead of swallowing `InterruptedException`. - `KafkaMaaSClient.watchTopicCreate` throws `IllegalStateException` after `close()`, instead of registering a callback that can never fire. + - Failed calls to maas-agent now throw `MaaSHttpException` instead of a bare `RuntimeException`. + It extends `MaaSException`, which is a `RuntimeException`, so existing `catch` blocks keep working. ## 10.0.0 * `Features` diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java index 0308e5b769..c7ff404d39 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java @@ -4,4 +4,9 @@ public class MaaSException extends RuntimeException { public MaaSException(String format, Object...args) { super(String.format(format, args)); } + + /** For subclasses whose message is already built and must not go through String.format. */ + protected MaaSException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java new file mode 100644 index 0000000000..1181b7ec68 --- /dev/null +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java @@ -0,0 +1,17 @@ +package com.netcracker.cloud.maas.client.api; + +/** + * A call to maas-agent that did not succeed: an unexpected status code, or a transport + * failure that outlived the retry budget. The message is taken as is, unlike + * {@link MaaSException}, because it carries request and response text. + */ +public class MaaSHttpException extends MaaSException { + + public MaaSHttpException(String message) { + super(message, (Throwable) null); + } + + public MaaSHttpException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index 8149d05aa8..9cd00a7881 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -216,10 +216,11 @@ private static Optional microProfileConfigOptional(String key) { Object config = getConfig.invoke(null); Method getOptionalValue = config.getClass().getMethod("getOptionalValue", String.class, Class.class); return (Optional) getOptionalValue.invoke(config, key, String.class); - } catch (ClassNotFoundException e) { + } catch (ClassNotFoundException | NoClassDefFoundError e) { + // MicroProfile Config is an optional dependency return Optional.empty(); - } catch (Throwable e) { - log.trace("MicroProfile Config not available or lookup failed for '{}'", key, e); + } catch (Exception e) { + log.trace("MicroProfile Config lookup failed for '{}'", key, e); return Optional.empty(); } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 566a64d828..39d4d6cddc 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.impl.Env; import lombok.extern.slf4j.Slf4j; import okhttp3.*; @@ -199,7 +200,7 @@ private static void sleepBackoff(int attempt, long maxTotalMillis, long deadline Thread.sleep(Math.min(backoffMillis(attempt, maxTotalMillis), remaining)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting to retry maas-agent request", e); + throw new MaaSHttpException("Interrupted while waiting to retry maas-agent request", e); } } @@ -258,7 +259,7 @@ private Optional sendAndReceive() { while (true) { long remainingMs = remainingMillis(deadlineNanos); if (remainingMs <= 0) { - throw new RuntimeException("Gave up on " + compiledReq + " after " + attempt + throw new MaaSHttpException("Gave up on " + compiledReq + " after " + attempt + " retries: the " + maxTotalMillis + "ms budget is spent"); } @@ -282,7 +283,7 @@ private Optional sendAndReceive() { sleepBackoff(attempt, maxTotalMillis, deadlineNanos); continue; } - throw new RuntimeException("Unexpected status code " + response.code() + throw new MaaSHttpException("Unexpected status code " + response.code() + " for request: " + compiledReq + giveUpSuffix(attempt) + "\n\tResponse body: " + errorBody); @@ -293,7 +294,7 @@ private Optional sendAndReceive() { return Optional.of(body); } catch (IOException e) { if (!canRetry(deadlineNanos)) { - throw new RuntimeException("Error executing " + compiledReq + giveUpSuffix(attempt), e); + throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempt), e); } attempt++; log.warn("Error execute http request: {}, Retry {}, within {}ms total", diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index e50badf25a..eb6f45c629 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -177,20 +177,100 @@ private void watchTenantCreateTopics() { if (closed) { return; } + if (closed || !parkUntilThereIsSomethingToWatch()) { + return; + } + } + } + /** + * Polls the watch endpoint until nothing is being watched any more. + * + * @return false if the thread must stop + */ + private boolean pollWhileThereIsSomethingToWatch() { + int failures = 0; + while (!closed && !topicCreateListeners.isEmpty()) { + String url = apiProvider.getKafkaTopicWatchCreateUrl(watchTimeout); + List found; try { - log.info("Nothing to watch, sleep thread."); - synchronized (watchLock) { - // guarded wait: a bare wait() would also return on a spurious wakeup - while (!closed && topicCreateListeners.isEmpty()) { - watchLock.wait(); - } + found = poll(url); + failures = 0; + } catch (Exception e) { + // `closed` is checked too: an interrupt can be swallowed further down + if (closed) { + return false; // shutting down, not a failure worth reporting } - log.info("Woke up!"); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; // exit loop + if (Thread.currentThread().isInterrupted()) { + log.warn("Watch thread interrupted without close(), stopping to watch {}", url, e); + return false; + } + failures++; + log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); + if (!sleepWatchBackoff(failures)) { + return false; // interrupted while backing off + } + continue; // nothing was received, nothing to deliver + } + deliver(found); + } + return true; + } + + /** One long poll for topics created since the previous call. */ + private List poll(String url) { + TypeReference> typeRef = new TypeReference<>() { + }; + return httpClient.request(url) + .post(topicCreateListeners.keySet()) + .expect(200) + .noRetry() + .sendAndReceive(typeRef) + .orElse(Collections.emptyList()); + } + + /** Hands each created topic to the callbacks registered for it, removing them as it goes. */ + private void deliver(List found) { + for (TopicInfo addr : found) { + List> callbacks = topicCreateListeners.remove(addr.getClassifier()); + if (callbacks == null) { + // this is unexpected situation in theory, but with this, code will be a little safer + continue; } + for (Consumer callback : callbacks) { + notifyCallback(addr, callback); + } + } + } + + private void notifyCallback(TopicInfo addr, Consumer callback) { + try { + log.info("Topic create event for {} received, execute callback {}", addr.getClassifier(), callback); + callback.accept(new TopicAddressImpl(addr)); + } catch (Exception e) { + log.error("Error execute callback {}", callback, e); + } + } + + /** + * Parks the thread while no topic is being watched. + * + * @return false if the thread was interrupted and must stop + */ + private boolean parkUntilThereIsSomethingToWatch() { + try { + log.info("Nothing to watch, sleep thread."); + synchronized (watchLock) { + // guarded wait: a bare wait() would also return on a spurious wakeup + while (!closed && topicCreateListeners.isEmpty()) { + watchLock.wait(); + } + } + log.info("Woke up!"); + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 51d9d6530c..4efd3415c5 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -1,5 +1,6 @@ package com.netcracker.cloud.maas.client.impl.http; +import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.impl.Env; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -15,7 +16,6 @@ import java.net.Socket; import java.time.Duration; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; @@ -26,6 +26,7 @@ import static com.netcracker.cloud.maas.client.Utils.withProp; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockserver.model.HttpRequest.request; @@ -102,10 +103,7 @@ void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); - withFastRetries(() -> - assertTrue(assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class) - ).getMessage().contains("401"))); + withFastRetries(() -> assertMessageContains("401", execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(HttpExecution.MAX_AUTH_RETRIES + 1)); @@ -121,10 +119,7 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(405).withBody("Method Not Allowed")); - withFastRetries(() -> - assertTrue(assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class) - ).getMessage().contains("405"))); + withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } @@ -134,46 +129,27 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { * agent cannot stretch the call past it. */ @Test - void testMaxTotalDuration_BoundsAHangingAttempt() throws Exception { + void testMaxTotalDuration_BoundsAHangingAttempt() throws IOException { // accepts the connection and never answers, unlike a refused connect which fails fast try (ServerSocket silentServer = new ServerSocket(0)) { - List accepted = Collections.synchronizedList(new ArrayList<>()); - Thread acceptor = new Thread(() -> { - try { - while (!silentServer.isClosed()) { - accepted.add(silentServer.accept()); - } - } catch (IOException e) { - // the socket was closed, the test is over - } - }, "silent-server"); - acceptor.setDaemon(true); - acceptor.start(); - - try { - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1000", () -> { - OkHttpClient client = new OkHttpClient.Builder() - .readTimeout(Duration.ofMinutes(1)) - .build(); - Request.Builder req = new Request.Builder() - .url("http://127.0.0.1:" + silentServer.getLocalPort() + PATH) - .get(); - HttpExecution execution = new HttpExecution(client, req).expect(200); - - long start = System.currentTimeMillis(); - assertThrows(RuntimeException.class, () -> execution.sendAndReceive(String.class)); - long elapsedMs = System.currentTimeMillis() - start; - assertTrue(elapsedMs < 20_000, - "expected the call to be bounded by its 1000ms budget rather than by the " - + "one minute read timeout, took " + elapsedMs + "ms"); - }); - } finally { - synchronized (accepted) { - for (Socket socket : accepted) { - socket.close(); - } - } - } + startAcceptor(silentServer, socket -> { /* hold the connection open and stay silent */ }); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1000", () -> { + OkHttpClient client = new OkHttpClient.Builder() + .readTimeout(Duration.ofMinutes(1)) + .build(); + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + silentServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(client, req).expect(200); + + long start = System.currentTimeMillis(); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + long elapsedMs = System.currentTimeMillis() - start; + assertTrue(elapsedMs < 20_000, + "expected the call to be bounded by its 1000ms budget rather than by the " + + "one minute read timeout, took " + elapsedMs + "ms"); + }); } } @@ -183,49 +159,55 @@ void testFailover_400NotRetried(ClientAndServer mockServer) { mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); - withFastRetries(() -> - assertTrue(assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class) - ).getMessage().contains("400"))); + withFastRetries(() -> assertMessageContains("400", execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } @Test - void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws Exception { - // A long total duration keeps the retry wait long enough for the interrupt to land in it. - withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "600000", () -> { - OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(Duration.ofMillis(300)) - .build(); - Request.Builder req = new Request.Builder().url("http://127.0.0.1:1/unreachable").get(); - HttpExecution execution = new HttpExecution(client, req); - execution.expect(200); - - AtomicBoolean interruptedAfter = new AtomicBoolean(); - AtomicReference thrown = new AtomicReference<>(); - CountDownLatch started = new CountDownLatch(1); - - Thread worker = new Thread(() -> { - started.countDown(); - try { - execution.sendAndReceive(String.class); - } catch (Throwable t) { - thrown.set(t); - } finally { - interruptedAfter.set(Thread.currentThread().isInterrupted()); - } - }, "http-execution-interrupt-test"); - worker.start(); - - assertTrue(started.await(2, TimeUnit.SECONDS)); - Thread.sleep(500); - worker.interrupt(); - worker.join(5000); - - assertFalse(worker.isAlive(), "worker should abort instead of continuing to retry after interrupt"); - assertTrue(interruptedAfter.get(), "interrupt flag must be restored after an interrupted retry wait"); - }); + void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { + // A server that drops every connection: the attempt fails at once and the loop moves + // into its backoff wait, which is where the interrupt has to land. + try (ServerSocket rudeServer = new ServerSocket(0)) { + CountDownLatch firstAttemptFailed = new CountDownLatch(1); + startAcceptor(rudeServer, socket -> { + socket.close(); + firstAttemptFailed.countDown(); + }); + + // A long total duration keeps the retry wait long enough for the interrupt to land in it. + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "600000", () -> { + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + rudeServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(new OkHttpClient(), req).expect(200); + + AtomicBoolean interruptedAfter = new AtomicBoolean(); + AtomicReference thrown = new AtomicReference<>(); + + Thread worker = new Thread(() -> { + try { + execution.sendAndReceive(String.class); + } catch (Exception e) { + thrown.set(e); + } finally { + interruptedAfter.set(Thread.currentThread().isInterrupted()); + } + }, "http-execution-interrupt-test"); + worker.start(); + + assertTrue(firstAttemptFailed.await(10, TimeUnit.SECONDS), "the first attempt never reached the server"); + worker.interrupt(); + worker.join(10_000); + + assertFalse(worker.isAlive(), "worker should abort instead of continuing to retry after interrupt"); + assertTrue(interruptedAfter.get(), "interrupt flag must be restored after an interrupted retry wait"); + assertInstanceOf(MaaSHttpException.class, thrown.get(), + "the interrupt must surface as a maas exception, not as an unrelated failure"); + assertTrue(thrown.get().getMessage().contains("Interrupted while waiting to retry"), + "unexpected message: " + thrown.get().getMessage()); + }); + } } // Delay must grow between attempts and saturate at a quarter of the total duration. @@ -252,9 +234,9 @@ void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServ .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "200", () -> { + HttpExecution execution = execution(mockServer).expect(200); long start = System.currentTimeMillis(); - assertThrows(RuntimeException.class, - () -> execution(mockServer).expect(200).sendAndReceive(String.class)); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); long elapsedMs = System.currentTimeMillis() - start; assertTrue(elapsedMs < 800, "expected retry loop to abort near the 200ms max total duration, took " + elapsedMs + "ms"); @@ -274,4 +256,42 @@ private static HttpExecution execution(ClientAndServer mockServer) { .get(); return new HttpExecution(client, req); } + + private static void assertMessageContains(String expected, HttpExecution execution) { + MaaSHttpException e = assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + assertTrue(e.getMessage().contains(expected), "unexpected message: " + e.getMessage()); + } + + @FunctionalInterface + private interface SocketHandler { + void handle(Socket socket) throws IOException; + } + + /** Serves the socket on a daemon thread until it is closed, then releases what it accepted. */ + private static void startAcceptor(ServerSocket server, SocketHandler handler) { + Thread acceptor = new Thread(() -> { + List accepted = new ArrayList<>(); + try { + while (!server.isClosed()) { + Socket socket = server.accept(); + accepted.add(socket); + handler.handle(socket); + } + } catch (IOException e) { + // the server socket was closed, the test is over + } finally { + accepted.forEach(HttpExecutionFailoverTest::closeQuietly); + } + }, "test-acceptor-" + server.getLocalPort()); + acceptor.setDaemon(true); + acceptor.start(); + } + + private static void closeQuietly(Socket socket) { + try { + socket.close(); + } catch (IOException e) { + // nothing useful to do while tearing a test down + } + } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index a0faf57aeb..c30d1cf83f 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -7,9 +7,11 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -35,16 +37,11 @@ class KafkaMaaSClientWatchBackoffTest { private static final String WATCHED_TOPIC = "orders"; private static final String NAMESPACE = "cloud-dev"; - /** - * The backoff is linear at one second per consecutive failure, so this window admits the - * first poll, a 1s pause, the second poll and a 2s pause. Anything much above that means - * the loop is not backing off at all. - */ - private static final long OBSERVATION_WINDOW_MILLIS = 2_500; - private static final int MAX_EXPECTED_POLLS = 5; + /** Three polls are enough to see the pause between them grow. */ + private static final int OBSERVED_POLLS = 3; - private final AtomicInteger watchPolls = new AtomicInteger(); - private final CountDownLatch firstPoll = new CountDownLatch(1); + private final List pollMillis = Collections.synchronizedList(new ArrayList<>()); + private final CountDownLatch pollsObserved = new CountDownLatch(OBSERVED_POLLS); private HttpServer agentStub; private KafkaMaaSClientImpl client; @@ -72,17 +69,18 @@ void failingWatchPollIsBackedOffInsteadOfHotLooping() { client = createKafkaClient(agentUrl); client.watchTopicCreate(WATCHED_TOPIC, addr -> { /* never created in this test */ }); - assertTrue(firstPoll.await(10, TimeUnit.SECONDS), - "the watch thread never reached the agent stub, so nothing was measured"); - Thread.sleep(OBSERVATION_WINDOW_MILLIS); - - int polls = watchPolls.get(); - // The lower bound matters as much as the upper one: without it the assertion - // would also pass when the loop never ran and nothing was verified. - assertTrue(polls >= 1, "watch loop did not poll at all, the test would pass vacuously"); - assertTrue(polls <= MAX_EXPECTED_POLLS, - "expected the watch loop to back off between failures, but it polled " + polls - + " times in " + OBSERVATION_WINDOW_MILLIS + "ms (limit " + MAX_EXPECTED_POLLS + ")"); + assertTrue(pollsObserved.await(30, TimeUnit.SECONDS), + "the watch loop reached the agent stub only " + pollMillis.size() + + " times out of " + OBSERVED_POLLS + ", so nothing was measured"); + + long firstPause = pollMillis.get(1) - pollMillis.get(0); + long secondPause = pollMillis.get(2) - pollMillis.get(1); + // A hot loop would show pauses near zero; a fixed delay would show two equal ones. + assertTrue(firstPause > 500, + "expected the watch loop to pause after a failure, but it polled again in " + firstPause + "ms"); + assertTrue(secondPause > firstPause, + "expected the pause to grow with consecutive failures, but got " + + firstPause + "ms then " + secondPause + "ms"); }); }); } @@ -93,13 +91,15 @@ private static KafkaMaaSClientImpl createKafkaClient(String agentUrl) { var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); - return new KafkaMaaSClientImpl(httpClient, null, new ApiUrlProvider(serverApiVersion, agentUrl)); + return new KafkaMaaSClientImpl(httpClient, + () -> { throw new UnsupportedOperationException("tenant manager is not used in this test"); }, + new ApiUrlProvider(serverApiVersion, agentUrl)); } /** Answers every poll with 500, the code maas-agent returns when it cannot reach maas-service. */ private void failWatchPoll(HttpExchange exchange) throws IOException { - watchPolls.incrementAndGet(); - firstPoll.countDown(); + pollMillis.add(System.currentTimeMillis()); + pollsObserved.countDown(); respond(exchange, 500, "{\"error\":\"error proxying request: maas-service unavailable\"}"); } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java index a374ca1ddf..048911468c 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -1,6 +1,7 @@ package com.netcracker.cloud.maas.client.impl.rabbit; import com.netcracker.cloud.maas.client.api.Classifier; +import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.api.rabbit.VHost; import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; import com.netcracker.cloud.maas.client.impl.Env; @@ -18,6 +19,7 @@ import static com.netcracker.cloud.maas.client.Utils.withProp; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; @@ -103,8 +105,8 @@ void testFailover_400NotRetried(ClientAndServer mockServer) { withProp(Env.PROP_NAMESPACE, "core-dev", () -> withFastRetries(() -> { RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); - org.junit.jupiter.api.Assertions.assertThrows(RuntimeException.class, - () -> client.getOrCreateVirtualHost(new Classifier("commands"))); + Classifier classifier = new Classifier("commands"); + assertThrows(MaaSHttpException.class, () -> client.getOrCreateVirtualHost(classifier)); })); mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(1)); From cb1e16096b7868c32db79d985e2bc65f91859d6c Mon Sep 17 00:00:00 2001 From: Ksiona Date: Tue, 11 Aug 2026 14:52:18 +0400 Subject: [PATCH 12/24] fix: total duration counter --- maas-client/README.md | 12 +-- .../cloud/maas/client/api/MaaSException.java | 5 ++ .../maas/client/api/MaaSHttpException.java | 2 +- .../cloud/maas/client/impl/Env.java | 27 +++++- .../maas/client/impl/http/HttpExecution.java | 51 ++++++++--- .../impl/kafka/KafkaMaaSClientImpl.java | 23 ++++- .../impl/http/HttpExecutionFailoverTest.java | 85 +++++++++++++++++-- .../KafkaMaaSClientWatchBackoffTest.java | 17 ++++ 8 files changed, 192 insertions(+), 30 deletions(-) diff --git a/maas-client/README.md b/maas-client/README.md index c1df3f53cd..5457d92243 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -66,24 +66,26 @@ setting: the maximum total duration of the call. | Property | Default | Meaning | |---|---|---| | `maas.http.timeout` | `30` (seconds) | connect/read/write timeout of a **single** attempt | -| `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included | +| `maas.http.retry.max-total-duration-ms` | `60000` | how long one call may take in **total**, retries included. `0` disables retries | `max-total-duration-ms` is the only retry knob: the attempt count and the pauses between attempts are derived from it. The first pause is 1s, each next one doubles, and the cap is a quarter of the total — with the default 60s that gives 1s, 2s, 4s, 8s, 15s, 15s, roughly six attempts when each attempt fails fast. If -attempts hang instead, fewer of them fit into the same budget. Backoff carries +attempts hang instead, fewer of them fit into the same duration. Backoff carries +/-20% jitter so concurrent callers do not retry in lockstep. Each attempt is additionally bounded by what is left of the total duration, so -the worst case a caller sees is the budget itself rather than the budget plus one -`maas.http.timeout`. +the worst case a caller sees is that total duration itself rather than the total +duration plus one `maas.http.timeout`. The 60s default is meant to outlast a database leader switchover while still failing fast enough to react to a real outage. The watch endpoint (`watch-create`) is excluded: it is a long poll with its own -loop and its own backoff. +loop and its own backoff. Its window is derived from `maas.http.timeout` and stays +below it — maas-service holds the request open for the whole window and then answers +with an empty list, which the client has to be able to receive. Which responses are retried: diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java index c7ff404d39..d0a16bfcd9 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java @@ -1,6 +1,11 @@ package com.netcracker.cloud.maas.client.api; public class MaaSException extends RuntimeException { + + /** + * Formats the message. Note that a two-argument call whose second argument is a + * {@code Throwable} binds to the constructor below instead, and is not formatted. + */ public MaaSException(String format, Object...args) { super(String.format(format, args)); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java index 1181b7ec68..69e1a470a7 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java @@ -2,7 +2,7 @@ /** * A call to maas-agent that did not succeed: an unexpected status code, or a transport - * failure that outlived the retry budget. The message is taken as is, unlike + * failure that outlived the configured total duration. The message is taken as is, unlike * {@link MaaSException}, because it carries request and response text. */ public class MaaSHttpException extends MaaSException { diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index 9cd00a7881..b3dc649894 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -110,20 +110,41 @@ public static Duration httpTimeout() { ); } + static final long DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS = 60_000L; + /** * How long one call to maas-agent may take in total, retries included. The only retry * knob: attempt count and backoff growth are derived from it. The 60s default outlasts * a database leader switchover. + *

+ * Zero disables retries, leaving a single attempt. An unreadable or negative value falls + * back to the default with a warning, rather than failing the call that happens to be first. */ public static Duration httpRetryMaxTotalDuration() { return Duration.ofMillis( stringProperty(PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS) - .map(Long::parseLong) - .filter(ms -> ms > 0) - .orElse(60_000L) + .map(Env::parseRetryDurationMillis) + .orElse(DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS) ); } + private static long parseRetryDurationMillis(String raw) { + long millis; + try { + millis = Long.parseLong(raw.trim()); + } catch (NumberFormatException e) { + log.warn("Ignoring '{}={}': not a number of milliseconds, using {}ms", + PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, raw, DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS); + return DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS; + } + if (millis < 0) { + log.warn("Ignoring '{}={}': negative, using {}ms", + PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, raw, DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS); + return DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS; + } + return millis; + } + public static String url2ws(String url) { return url.replaceAll("^http(s?):", "ws$1:"); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 39d4d6cddc..095ad6c3b8 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -127,14 +127,16 @@ private static boolean isRetryableStatus(int code, String body) { /** * Recognises the 405 that maas-service returns for PostgreSQL error 25006, mapped from - * {@code DatabaseIsReadonlyError} / {@code DatabaseIsNotActiveError}. Matched on the - * reason text because the TMF code is the same for every maas-service error. + * {@code DatabaseIsReadonlyError} ("database is in read-only mode") and + * {@code DatabaseIsNotActiveError} ("database is not in 'active' mode"), both declared in + * maas-service and mapped to 405. */ private static boolean isDatabaseUnavailable(String body) { if (body == null || !body.contains(MAAS_ERROR_CODE)) { return false; } - return body.contains("read-only") || body.contains("not in 'active' mode"); + String reason = body.toLowerCase(Locale.ROOT); + return reason.contains("read-only") || reason.contains("read only") || reason.contains("active"); } /** @@ -146,7 +148,8 @@ private static boolean isDatabaseUnavailable(String body) { private int authAttempts = 0; - private boolean canRetryStatus(int code, String body, long deadlineNanos) { + /** Decides on a retry and counts the 401 attempt, so it is called once per response. */ + private boolean takeRetrySlotFor(int code, String body, long deadlineNanos) { if (!isRetryableStatus(code, body) || !canRetry(deadlineNanos)) { return false; } @@ -234,13 +237,14 @@ private static String giveUpSuffix(int attempt) { /** * Client for one attempt, bounded by what is left of the total duration. Without it an * attempt starting just before the deadline still runs for the full - * {@code maas.http.timeout} and the call overruns its budget. + * {@code maas.http.timeout} and the call overruns its total duration. *

* Not applied under {@link #noRetry()}: there the caller owns the lifecycle, and the - * watch long poll legitimately runs as long as the budget itself. + * watch long poll legitimately runs as long as the total duration itself. */ private OkHttpClient clientForAttempt(long remainingMs) { - if (!retryEnabled) { + if (!retryEnabled || remainingMs <= 0) { + // no retries, or a zero total duration: the single attempt keeps the client's own timeouts return httpClient; } // newBuilder shares the connection pool and dispatcher, so this is cheap @@ -256,11 +260,16 @@ private Optional sendAndReceive() { long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); int attempt = 0; + authAttempts = 0; + // what made the previous attempt fail, so that giving up reports the cause and not + // just the elapsed time + Throwable lastFailure = null; + String lastStatusAndBody = null; while (true) { long remainingMs = remainingMillis(deadlineNanos); - if (remainingMs <= 0) { - throw new MaaSHttpException("Gave up on " + compiledReq + " after " + attempt - + " retries: the " + maxTotalMillis + "ms budget is spent"); + // the total duration bounds retries, not the call: the first attempt always goes out + if (attempt > 0 && remainingMs <= 0) { + throw totalDurationExceeded(compiledReq, attempt, maxTotalMillis, lastStatusAndBody, lastFailure); } try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { @@ -276,8 +285,10 @@ private Optional sendAndReceive() { // read once, without throwing: a body that cannot be read must not turn a // permanent status into a retry String errorBody = errorBodyOrPlaceholder(response); - if (canRetryStatus(response.code(), errorBody, deadlineNanos)) { + if (takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { attempt++; + lastFailure = null; + lastStatusAndBody = "status " + response.code() + ", body: " + errorBody; log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", response.code(), compiledReq, attempt, maxTotalMillis); sleepBackoff(attempt, maxTotalMillis, deadlineNanos); @@ -297,10 +308,28 @@ private Optional sendAndReceive() { throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempt), e); } attempt++; + lastFailure = e; + lastStatusAndBody = null; log.warn("Error execute http request: {}, Retry {}, within {}ms total", e.getMessage(), attempt, maxTotalMillis); sleepBackoff(attempt, maxTotalMillis, deadlineNanos); } } } + + /** + * The usual terminal failure: the backoff is clamped to the time left, so a call that keeps + * failing lands exactly on the deadline. Carries what the last attempt saw, otherwise the + * trace says only that a minute went by. + */ + private static MaaSHttpException totalDurationExceeded(Request req, int attempt, long maxTotalMillis, + String lastStatusAndBody, Throwable lastFailure) { + String message = "Gave up on " + req + " after " + attempt + " retries: ran out of its " + + maxTotalMillis + "ms total duration." + + "\n\tLast attempt: " + (lastStatusAndBody != null ? lastStatusAndBody + : lastFailure != null ? lastFailure.toString() : "unknown"); + return lastFailure != null + ? new MaaSHttpException(message, lastFailure) + : new MaaSHttpException(message); + } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index eb6f45c629..d129e89d84 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -23,6 +23,7 @@ import com.netcracker.cloud.maas.client.api.kafka.TopicAddress; import com.netcracker.cloud.maas.client.api.kafka.TopicCreateOptions; import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; +import com.netcracker.cloud.maas.client.impl.Env; import com.netcracker.cloud.maas.client.impl.Lazy; import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicDeleteRequest; import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicDeleteResponse; @@ -40,7 +41,20 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { private final Lazy tenantManagerConnector; private final ApiUrlProvider apiProvider; - private final Duration watchTimeout = Duration.ofSeconds(60); + /** + * How long maas-service is asked to hold a watch poll open. It must stay below the client's + * own read timeout, otherwise every quiet poll dies locally instead of returning an empty + * 200 — which counts as a failure and walks the backoff up to its cap, delaying the next + * real topic-create event. maas-service caps the window at 120s in any case. + */ + private final Duration watchTimeout = watchTimeout(Env.httpTimeout()); + + static Duration watchTimeout(Duration httpTimeout) { + Duration margin = Duration.ofSeconds(5); + Duration window = httpTimeout.minus(margin); + return window.compareTo(margin) < 0 ? margin : window; + } + private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); private static final Duration WATCH_MAX_RETRY_INTERVAL = Duration.ofSeconds(30); // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time @@ -217,15 +231,16 @@ private boolean pollWhileThereIsSomethingToWatch() { return true; } + private static final TypeReference> TOPIC_LIST = new TypeReference<>() { + }; + /** One long poll for topics created since the previous call. */ private List poll(String url) { - TypeReference> typeRef = new TypeReference<>() { - }; return httpClient.request(url) .post(topicCreateListeners.keySet()) .expect(200) .noRetry() - .sendAndReceive(typeRef) + .sendAndReceive(TOPIC_LIST) .orElse(Collections.emptyList()); } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 4efd3415c5..95860eb5f5 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -94,7 +94,7 @@ void testFailover_401ThenSuccess(ClientAndServer mockServer) { /** * A 401 that keeps coming back means the supplier is handing out a token the server * rejects, and it has no way of being told so. Further attempts resend the same token, - * so the budget is deliberately tighter than the overall duration: a wrong secret must + * so retrying it is deliberately capped tighter than the total duration: a wrong secret must * fail fast instead of hanging for the whole minute. */ @Test @@ -125,7 +125,7 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { } /** - * The per-attempt budget is clamped to what is left of the total duration, so a hanging + * Each attempt is clamped to what is left of the total duration, so a hanging * agent cannot stretch the call past it. */ @Test @@ -146,13 +146,62 @@ void testMaxTotalDuration_BoundsAHangingAttempt() throws IOException { long start = System.currentTimeMillis(); assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); long elapsedMs = System.currentTimeMillis() - start; - assertTrue(elapsedMs < 20_000, - "expected the call to be bounded by its 1000ms budget rather than by the " + assertTrue(elapsedMs < 5_000, + "expected the call to be bounded by its 1000ms total duration rather than by the " + "one minute read timeout, took " + elapsedMs + "ms"); }); } } + @Test + void testFailover_429Retried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.exactly(1)) + .respond(response().withStatusCode(429).withBody("{\"error\":\"slow down\"}")); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(200).withBody("\"ok\"")); + + withFastRetries(() -> { + Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); + assertEquals("ok", body.orElseThrow()); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(2)); + } + + /** The watch long poll owns its own loop, so its execution must send the request exactly once. */ + @Test + void testNoRetry_SendsExactlyOneAttempt(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + withFastRetries(() -> { + HttpExecution execution = execution(mockServer).expect(200).noRetry(); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + + /** + * A zero total duration is the config-level off switch: one attempt, no retries, and no + * per-attempt clamp that would cut that attempt short. + */ + @Test + void testZeroTotalDuration_SendsOneAttemptAndDoesNotRetry(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "0", () -> { + HttpExecution execution = execution(mockServer).expect(200); + assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); + }); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + @Test void testFailover_400NotRetried(ClientAndServer mockServer) { mockServer.reset(); @@ -210,19 +259,43 @@ void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { } } + /** + * Running out of time is the usual way a failover call ends, so the exception has to say what + * kept failing. Without the cause the trace shows only that a minute went by. + */ + @Test + void testTotalDurationExceeded_CarriesTheLastFailureAsCause() throws IOException { + try (ServerSocket rudeServer = new ServerSocket(0)) { + startAcceptor(rudeServer, Socket::close); + + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1500", () -> { + Request.Builder req = new Request.Builder() + .url("http://127.0.0.1:" + rudeServer.getLocalPort() + PATH) + .get(); + HttpExecution execution = new HttpExecution(new OkHttpClient(), req).expect(200); + + MaaSHttpException e = assertThrows(MaaSHttpException.class, + () -> execution.sendAndReceive(String.class)); + assertTrue(e.getMessage().contains("ran out of its"), "unexpected message: " + e.getMessage()); + assertInstanceOf(IOException.class, e.getCause(), + "the transport failure that consumed the time must be the cause"); + }); + } + } + // Delay must grow between attempts and saturate at a quarter of the total duration. @Test void testBackoffMillis_GrowsAndSaturatesAtTheCap() { long[] expectedFor60s = {1_000, 2_000, 4_000, 8_000, 15_000, 15_000}; for (int attempt = 1; attempt <= expectedFor60s.length; attempt++) { assertEquals(expectedFor60s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 60_000), - "attempt " + attempt + " of a 60s budget"); + "attempt " + attempt + " of a 60s total duration"); } long[] expectedFor5s = {1_000, 1_250, 1_250}; for (int attempt = 1; attempt <= expectedFor5s.length; attempt++) { assertEquals(expectedFor5s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 5_000), - "attempt " + attempt + " of a 5s budget"); + "attempt " + attempt + " of a 5s total duration"); } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index c30d1cf83f..70f213eb59 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -1,9 +1,11 @@ package com.netcracker.cloud.maas.client.impl.kafka; import static com.netcracker.cloud.maas.client.Utils.withProp; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; +import java.time.Duration; import java.io.OutputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; @@ -61,6 +63,21 @@ void stopClientAndStub() { agentStub.stop(0); } + /** + * maas-service holds a watch poll open for the whole requested window and then answers 200 + * with an empty list. If the window outlasts the client read timeout, that answer never + * arrives: every quiet poll fails locally, walks the backoff up to its 30s cap and delays + * the next real topic-create event. + */ + @Test + void watchWindowStaysBelowTheReadTimeout() { + assertTrue(KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)).compareTo(Duration.ofSeconds(30)) < 0, + "the watch window must leave the read timeout room to receive the answer"); + assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30))); + // a read timeout too small to leave a margin still yields a usable window + assertEquals(Duration.ofSeconds(5), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(2))); + } + @Test void failingWatchPollIsBackedOffInsteadOfHotLooping() { withProp(Env.PROP_NAMESPACE, NAMESPACE, () -> { From ab892dff5d7540b7ca0ab4e72ca82ecd27580a90 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Wed, 12 Aug 2026 10:39:06 +0400 Subject: [PATCH 13/24] fix: sonar issues and documentation --- .../blue-green-state-monitor-java/pom.xml | 5 + .../cloud/bluegreen/AbstractBGTest.java | 32 +++--- maas-client/CHANGELOG.md | 6 + maas-client/README.md | 3 +- .../maas/client/impl/http/HttpExecution.java | 106 +++++++++++------- .../impl/kafka/KafkaMaaSClientImpl.java | 14 ++- .../impl/http/HttpExecutionFailoverTest.java | 16 +++ .../KafkaMaaSClientWatchBackoffTest.java | 15 ++- 8 files changed, 132 insertions(+), 65 deletions(-) diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml index f9508fa858..6946560c72 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml @@ -56,6 +56,11 @@ junit-jupiter test + + org.awaitility + awaitility + test + com.squareup.okhttp3 okhttp diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java index c3be442445..d26a411196 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java @@ -2,6 +2,7 @@ import com.netcracker.cloud.bluegreen.impl.http.HttpClientAdapter; import lombok.SneakyThrows; +import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeEach; import org.testcontainers.consul.ConsulContainer; import org.testcontainers.containers.wait.strategy.Wait; @@ -15,8 +16,9 @@ import java.util.concurrent.Callable; import java.util.function.Supplier; +/** Shared fixture: a Consul container per test method, plus small waiting helpers. */ @Testcontainers -class AbstractBGTest { +abstract class AbstractBGTest { String ns1 = "ns-1"; String ns2 = "ns-2"; @@ -50,23 +52,17 @@ void before() { * cannot be bound to a node that is not there yet: consul replies 500 "Missing node registration". */ private void awaitNodeRegistered() { - Instant deadline = Instant.now().plus(NODE_REGISTRATION_TIMEOUT); - String lastSeen = "no response"; - while (Instant.now().isBefore(deadline)) { - try { - String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), - String.class).sendAndGet(); - lastSeen = nodes; - if (nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]")) { - return; - } - } catch (Exception e) { - lastSeen = e.toString(); - } - run(() -> Thread.sleep(100)); - } - throw new IllegalStateException("Consul node was not registered in the catalog within " - + NODE_REGISTRATION_TIMEOUT + ", last response: " + lastSeen); + Awaitility.await("consul node registered in the catalog") + .atMost(NODE_REGISTRATION_TIMEOUT) + .pollInterval(Duration.ofMillis(100)) + .ignoreExceptions() + .until(this::catalogHasNodes); + } + + private boolean catalogHasNodes() { + String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), + String.class).sendAndGet(); + return nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]"); } @SneakyThrows diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 4ba81264e4..ca5ebb86ea 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -24,6 +24,12 @@ registering a callback that can never fire. - Failed calls to maas-agent now throw `MaaSHttpException` instead of a bare `RuntimeException`. It extends `MaaSException`, which is a `RuntimeException`, so existing `catch` blocks keep working. + Note the widening: `catch (MaaSException)` used to mean a MaaS business error and now also + catches transport failures, such as the agent being unreachable for the whole minute. + - `maas.http.retry.max-total-duration-ms=0` disables retries, leaving a single attempt. An + unreadable or negative value logs a warning and falls back to the 60s default. + - The Kafka watch poll window is derived from `maas.http.timeout` (25s with the defaults) instead + of a fixed 60s that outlasted the read timeout, so tuning `maas.http.timeout` now also moves it. ## 10.0.0 * `Features` diff --git a/maas-client/README.md b/maas-client/README.md index 5457d92243..a075de24bb 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -85,7 +85,8 @@ failing fast enough to react to a real outage. The watch endpoint (`watch-create`) is excluded: it is a long poll with its own loop and its own backoff. Its window is derived from `maas.http.timeout` and stays below it — maas-service holds the request open for the whole window and then answers -with an empty list, which the client has to be able to receive. +with an empty list, which the client has to be able to receive. With the default 30s +timeout the window is 25s. Which responses are retried: diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 095ad6c3b8..fd29f63d67 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -136,7 +136,8 @@ private static boolean isDatabaseUnavailable(String body) { return false; } String reason = body.toLowerCase(Locale.ROOT); - return reason.contains("read-only") || reason.contains("read only") || reason.contains("active"); + return reason.contains("read-only") || reason.contains("read only") + || reason.contains("not in 'active' mode"); } /** @@ -259,17 +260,12 @@ private Optional sendAndReceive() { long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); - int attempt = 0; + Attempts attempts = new Attempts(); authAttempts = 0; - // what made the previous attempt fail, so that giving up reports the cause and not - // just the elapsed time - Throwable lastFailure = null; - String lastStatusAndBody = null; while (true) { long remainingMs = remainingMillis(deadlineNanos); - // the total duration bounds retries, not the call: the first attempt always goes out - if (attempt > 0 && remainingMs <= 0) { - throw totalDurationExceeded(compiledReq, attempt, maxTotalMillis, lastStatusAndBody, lastFailure); + if (attempts.outOfTime(remainingMs)) { + throw totalDurationExceeded(compiledReq, attempts, maxTotalMillis); } try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { @@ -282,22 +278,8 @@ private Optional sendAndReceive() { } if (!expectedCodes.contains(response.code())) { - // read once, without throwing: a body that cannot be read must not turn a - // permanent status into a retry - String errorBody = errorBodyOrPlaceholder(response); - if (takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { - attempt++; - lastFailure = null; - lastStatusAndBody = "status " + response.code() + ", body: " + errorBody; - log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", - response.code(), compiledReq, attempt, maxTotalMillis); - sleepBackoff(attempt, maxTotalMillis, deadlineNanos); - continue; - } - throw new MaaSHttpException("Unexpected status code " + response.code() - + " for request: " + compiledReq - + giveUpSuffix(attempt) - + "\n\tResponse body: " + errorBody); + retryOrFail(response, compiledReq, attempts, deadlineNanos, maxTotalMillis); + continue; } String body = bodyAsString(response); @@ -305,31 +287,79 @@ private Optional sendAndReceive() { return Optional.of(body); } catch (IOException e) { if (!canRetry(deadlineNanos)) { - throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempt), e); + throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempts.count), e); } - attempt++; - lastFailure = e; - lastStatusAndBody = null; + attempts.afterTransportError(e); log.warn("Error execute http request: {}, Retry {}, within {}ms total", - e.getMessage(), attempt, maxTotalMillis); - sleepBackoff(attempt, maxTotalMillis, deadlineNanos); + e.getMessage(), attempts.count, maxTotalMillis); + sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); } } } + /** + * Handles a status the caller did not expect: waits before the next attempt, or throws when + * the status is terminal. + */ + private void retryOrFail(Response response, Request compiledReq, Attempts attempts, + long deadlineNanos, long maxTotalMillis) { + // read once, without throwing: a body that cannot be read must not turn a permanent + // status into a retry + String errorBody = errorBodyOrPlaceholder(response); + if (!takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { + throw new MaaSHttpException("Unexpected status code " + response.code() + + " for request: " + compiledReq + + giveUpSuffix(attempts.count) + + "\n\tResponse body: " + errorBody); + } + attempts.afterStatus(response.code(), errorBody); + log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", + response.code(), compiledReq, attempts.count, maxTotalMillis); + sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); + } + + /** How many attempts went out and what the last one failed with. */ + private static final class Attempts { + private int count; + private Throwable lastFailure; + private String lastStatusAndBody; + + /** The total duration bounds retries, not the call: the first attempt always goes out. */ + boolean outOfTime(long remainingMs) { + return count > 0 && remainingMs <= 0; + } + + void afterStatus(int code, String body) { + count++; + lastFailure = null; + lastStatusAndBody = "status " + code + ", body: " + body; + } + + void afterTransportError(IOException e) { + count++; + lastFailure = e; + lastStatusAndBody = null; + } + + String describeLast() { + if (lastStatusAndBody != null) { + return lastStatusAndBody; + } + return lastFailure != null ? lastFailure.toString() : "unknown"; + } + } + /** * The usual terminal failure: the backoff is clamped to the time left, so a call that keeps * failing lands exactly on the deadline. Carries what the last attempt saw, otherwise the * trace says only that a minute went by. */ - private static MaaSHttpException totalDurationExceeded(Request req, int attempt, long maxTotalMillis, - String lastStatusAndBody, Throwable lastFailure) { - String message = "Gave up on " + req + " after " + attempt + " retries: ran out of its " + private static MaaSHttpException totalDurationExceeded(Request req, Attempts attempts, long maxTotalMillis) { + String message = "Gave up on " + req + " after " + attempts.count + " retries: ran out of its " + maxTotalMillis + "ms total duration." - + "\n\tLast attempt: " + (lastStatusAndBody != null ? lastStatusAndBody - : lastFailure != null ? lastFailure.toString() : "unknown"); - return lastFailure != null - ? new MaaSHttpException(message, lastFailure) + + "\n\tLast attempt: " + attempts.describeLast(); + return attempts.lastFailure != null + ? new MaaSHttpException(message, attempts.lastFailure) : new MaaSHttpException(message); } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index d129e89d84..438d36dd79 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -49,10 +49,18 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { */ private final Duration watchTimeout = watchTimeout(Env.httpTimeout()); + /** Largest gap left between the watch window and the read timeout. */ + private static final long MAX_WATCH_MARGIN_SECONDS = 5; + + /** + * The margin is clamped rather than subtracted outright, so that a small read timeout + * narrows the window instead of pushing it past the timeout. Whole seconds, because that + * is how the window travels in the query string. + */ static Duration watchTimeout(Duration httpTimeout) { - Duration margin = Duration.ofSeconds(5); - Duration window = httpTimeout.minus(margin); - return window.compareTo(margin) < 0 ? margin : window; + long timeoutSeconds = httpTimeout.getSeconds(); + long marginSeconds = Math.min(MAX_WATCH_MARGIN_SECONDS, timeoutSeconds / 2); + return Duration.ofSeconds(Math.max(1, timeoutSeconds - marginSeconds)); } private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 95860eb5f5..fb325707d1 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -124,6 +124,22 @@ void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } + /** + * The envelope alone does not make a 405 transient: every maas-service error carries the same + * code, so the reason has to name the read-only database and not merely contain its words. + */ + @Test + void testFailover_405WithUnrelatedMaasReasonNotRetried(ClientAndServer mockServer) { + mockServer.reset(); + mockServer.when(request().withPath(PATH), Times.unlimited()) + .respond(response().withStatusCode(405) + .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}")); + + withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); + + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + } + /** * Each attempt is clamped to what is left of the total duration, so a hanging * agent cannot stretch the call past it. diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index 70f213eb59..f5521af4ef 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -71,11 +71,16 @@ void stopClientAndStub() { */ @Test void watchWindowStaysBelowTheReadTimeout() { - assertTrue(KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)).compareTo(Duration.ofSeconds(30)) < 0, - "the watch window must leave the read timeout room to receive the answer"); - assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30))); - // a read timeout too small to leave a margin still yields a usable window - assertEquals(Duration.ofSeconds(5), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(2))); + // the invariant, checked across the range rather than at one point + for (long readTimeoutSeconds : new long[]{2, 5, 6, 10, 30, 60, 120}) { + Duration window = KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(readTimeoutSeconds)); + assertTrue(window.getSeconds() < readTimeoutSeconds, + "a " + readTimeoutSeconds + "s read timeout must leave room for the answer, got " + window); + assertTrue(window.getSeconds() >= 1, + "the window travels in whole seconds, so it must not round down to zero: " + window); + } + assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)), + "the default read timeout should keep the full margin"); } @Test From 46d4ee3c1484cd7383d4c8361791f320d7048fe3 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Wed, 12 Aug 2026 13:23:34 +0400 Subject: [PATCH 14/24] fix: added parametrized test for several similar ones --- .../impl/http/HttpExecutionFailoverTest.java | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index fb325707d1..685d094085 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -6,6 +6,9 @@ import okhttp3.Request; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockserver.integration.ClientAndServer; import org.mockserver.junit.jupiter.MockServerExtension; import org.mockserver.matchers.Times; @@ -22,6 +25,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import static com.netcracker.cloud.maas.client.Utils.withProp; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -29,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; @@ -109,33 +114,29 @@ void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { VerificationTimes.exactly(HttpExecution.MAX_AUTH_RETRIES + 1)); } - /** - * A 405 without a maas-service error envelope is an ordinary "method not allowed" — - * a route or an ingress rejecting the request — and must fail fast. - */ - @Test - void testFailover_405WithoutMaasEnvelopeNotRetried(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(405).withBody("Method Not Allowed")); - - withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); - - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); + /** Responses that are permanent, so the call must fail on its first attempt. */ + static Stream permanentResponses() { + return Stream.of( + arguments("a plain client error", 400, "{\"error\":\"bad request\"}"), + // 405 is transient only for a read-only database; a route removed on the server + // or an ingress rejecting the method is not + arguments("405 without a maas-service envelope", 405, "Method Not Allowed"), + // every maas-service error carries MAAS-0600, so the envelope alone means nothing: + // the reason has to name the read-only database, not merely contain its words + arguments("405 whose maas-service reason is unrelated", 405, + "{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}") + ); } - /** - * The envelope alone does not make a 405 transient: every maas-service error carries the same - * code, so the reason has to name the read-only database and not merely contain its words. - */ - @Test - void testFailover_405WithUnrelatedMaasReasonNotRetried(ClientAndServer mockServer) { + @ParameterizedTest(name = "{0} is not retried") + @MethodSource("permanentResponses") + void testFailover_PermanentResponseNotRetried(String description, int status, String body, + ClientAndServer mockServer) { mockServer.reset(); mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(405) - .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}")); + .respond(response().withStatusCode(status).withBody(body)); - withFastRetries(() -> assertMessageContains("405", execution(mockServer).expect(200))); + withFastRetries(() -> assertMessageContains(String.valueOf(status), execution(mockServer).expect(200))); mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } @@ -218,17 +219,6 @@ void testZeroTotalDuration_SendsOneAttemptAndDoesNotRetry(ClientAndServer mockSe mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); } - @Test - void testFailover_400NotRetried(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(400).withBody("{\"error\":\"bad request\"}")); - - withFastRetries(() -> assertMessageContains("400", execution(mockServer).expect(200))); - - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(1)); - } - @Test void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { // A server that drops every connection: the attempt fails at once and the loop moves From 2f60016ec3be3102dd4094b897753ae239e50346 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 11:24:45 +0400 Subject: [PATCH 15/24] chore: refactoring, failsafe package were added --- .../blue-green-state-monitor-java/pom.xml | 5 - .../cloud/bluegreen/AbstractBGTest.java | 26 +- maas-client/client/pom.xml | 4 + .../maas/client/impl/http/HttpExecution.java | 249 ++++++++---------- .../impl/http/HttpExecutionFailoverTest.java | 16 -- maas-client/pom.xml | 6 + 6 files changed, 117 insertions(+), 189 deletions(-) diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml index 6946560c72..f9508fa858 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/pom.xml @@ -56,11 +56,6 @@ junit-jupiter test - - org.awaitility - awaitility - test - com.squareup.okhttp3 okhttp diff --git a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java index d26a411196..495c218c5a 100644 --- a/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java +++ b/core-blue-green-state-monitor/blue-green-state-monitor-java/src/test/java/com/netcracker/cloud/bluegreen/AbstractBGTest.java @@ -2,23 +2,20 @@ import com.netcracker.cloud.bluegreen.impl.http.HttpClientAdapter; import lombok.SneakyThrows; -import org.awaitility.Awaitility; import org.junit.jupiter.api.BeforeEach; import org.testcontainers.consul.ConsulContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.concurrent.Callable; import java.util.function.Supplier; -/** Shared fixture: a Consul container per test method, plus small waiting helpers. */ @Testcontainers -abstract class AbstractBGTest { +class AbstractBGTest { String ns1 = "ns-1"; String ns2 = "ns-2"; @@ -33,8 +30,6 @@ abstract class AbstractBGTest { String consulUrl; - private static final Duration NODE_REGISTRATION_TIMEOUT = Duration.ofSeconds(30); - @Container ConsulContainer consulContainer = new ConsulContainer("hashicorp/consul:1.16") .waitingFor(Wait.forHttp("/v1/catalog/nodes") @@ -44,25 +39,6 @@ abstract class AbstractBGTest { @BeforeEach void before() { consulUrl = String.format("http://%s:%d", consulContainer.getHost(), consulContainer.getMappedPort(8500)); - awaitNodeRegistered(); - } - - /** - * The agent answers on its port before it has registered itself in the catalog, and a session - * cannot be bound to a node that is not there yet: consul replies 500 "Missing node registration". - */ - private void awaitNodeRegistered() { - Awaitility.await("consul node registered in the catalog") - .atMost(NODE_REGISTRATION_TIMEOUT) - .pollInterval(Duration.ofMillis(100)) - .ignoreExceptions() - .until(this::catalogHasNodes); - } - - private boolean catalogHasNodes() { - String nodes = client.invoke(req -> req.uri(URI.create(consulUrl + "/v1/catalog/nodes")).GET(), - String.class).sendAndGet(); - return nodes != null && !nodes.isBlank() && !nodes.strip().equals("[]"); } @SneakyThrows diff --git a/maas-client/client/pom.xml b/maas-client/client/pom.xml index ecb2628e22..8a3e6e6795 100644 --- a/maas-client/client/pom.xml +++ b/maas-client/client/pom.xml @@ -26,6 +26,10 @@ lombok provided + + dev.failsafe + failsafe + com.squareup.okhttp3 diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index fd29f63d67..0392146641 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -6,14 +6,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.impl.Env; +import dev.failsafe.ExecutionContext; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; +import dev.failsafe.RetryPolicyBuilder; import lombok.extern.slf4j.Slf4j; import okhttp3.*; import java.io.IOException; import java.time.Duration; import java.util.*; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -149,64 +152,15 @@ private static boolean isDatabaseUnavailable(String body) { private int authAttempts = 0; - /** Decides on a retry and counts the 401 attempt, so it is called once per response. */ - private boolean takeRetrySlotFor(int code, String body, long deadlineNanos) { - if (!isRetryableStatus(code, body) || !canRetry(deadlineNanos)) { - return false; - } - if (code == 401) { - return ++authAttempts <= MAX_AUTH_RETRIES; - } - return true; + /** Asked once per failed attempt, so the 401 is counted here. */ + private boolean worthAnotherAttempt(RetryableStatus status) { + return status.code != 401 || ++authAttempts <= MAX_AUTH_RETRIES; } - /** First backoff pause. */ - private static final long BASE_DELAY_MILLIS = 1_000L; - - /** A single pause is capped at this fraction of the total duration. */ + /** First backoff pause, and the fraction of the total duration a single pause may reach. */ + private static final Duration BASE_DELAY = Duration.ofSeconds(1); private static final int MAX_DELAY_FRACTION_OF_TOTAL = 4; - - /** - * Delay before jitter: doubles per attempt, capped. Integer arithmetic saturating at - * the cap, so a large attempt count cannot overflow. - */ - static long cappedDelayMillis(int attempt, long maxTotalMillis) { - long max = Math.max(1L, maxTotalMillis / MAX_DELAY_FRACTION_OF_TOTAL); - long delay = Math.min(BASE_DELAY_MILLIS, max); - for (int i = 1; i < attempt && delay < max; i++) { - delay = delay > max / 2 ? max : delay * 2; - } - return delay; - } - - // Exponential backoff with jitter between retries. - private static long backoffMillis(int attempt, long maxTotalMillis) { - long capped = cappedDelayMillis(attempt, maxTotalMillis); - double jitterFactor = 0.8 + ThreadLocalRandom.current().nextDouble() * 0.4; - return Math.max(1L, (long) (capped * jitterFactor)); - } - - // The total duration is the only stop condition, unless noRetry() was used. - private boolean canRetry(long deadlineNanos) { - return retryEnabled && System.nanoTime() < deadlineNanos; - } - - /** - * Waits before the next retry, clamped to what is left of the total duration so the - * backoff cannot overshoot it. Restores the interrupt flag and aborts if interrupted. - */ - private static void sleepBackoff(int attempt, long maxTotalMillis, long deadlineNanos) { - long remaining = remainingMillis(deadlineNanos); - if (remaining <= 0) { - return; - } - try { - Thread.sleep(Math.min(backoffMillis(attempt, maxTotalMillis), remaining)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new MaaSHttpException("Interrupted while waiting to retry maas-agent request", e); - } - } + private static final double JITTER = 0.2; // Response.body() is nullable in OkHttp; a missing body reads as empty. private static String bodyAsString(Response response) throws IOException { @@ -227,10 +181,6 @@ private static String errorBodyOrPlaceholder(Response response) { } } - private static long remainingMillis(long deadlineNanos) { - return TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); - } - private static String giveUpSuffix(int attempt) { return attempt == 0 ? "" : ", gave up after " + attempt + " retries"; } @@ -254,112 +204,125 @@ private OkHttpClient clientForAttempt(long remainingMs) { .build(); } + /** + * Backoff, jitter, attempt counting and the overall deadline belong to the retry policy; + * what is left here is what one attempt is and which of its outcomes is worth repeating. + */ private Optional sendAndReceive() { Request compiledReq = req.build(); log.debug("Send request: {}", compiledReq); long maxTotalMillis = Env.httpRetryMaxTotalDuration().toMillis(); - long deadlineNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(maxTotalMillis); - Attempts attempts = new Attempts(); + if (!retryEnabled || maxTotalMillis <= 0) { + return attemptOnce(compiledReq); + } authAttempts = 0; - while (true) { - long remainingMs = remainingMillis(deadlineNanos); - if (attempts.outOfTime(remainingMs)) { - throw totalDurationExceeded(compiledReq, attempts, maxTotalMillis); + try { + return Failsafe.with(retryPolicy(compiledReq, maxTotalMillis)).get(context -> + attempt(compiledReq, maxTotalMillis - context.getElapsedTime().toMillis(), context)); + } catch (RetryableStatus e) { + throw exhausted(compiledReq, maxTotalMillis, e.describe(), null); + } catch (FailsafeException e) { + if (e.getCause() instanceof InterruptedException interrupted) { + // Failsafe restores the flag; the call still has to abort rather than retry + Thread.currentThread().interrupt(); + throw new MaaSHttpException("Interrupted while waiting to retry maas-agent request", interrupted); } + throw exhausted(compiledReq, maxTotalMillis, String.valueOf(e.getCause()), e.getCause()); + } + } - try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { - // check response codes against acceptable list - log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); + private RetryPolicy> retryPolicy(Request compiledReq, long maxTotalMillis) { + Duration maxDelay = Duration.ofMillis(Math.max(1, maxTotalMillis / MAX_DELAY_FRACTION_OF_TOTAL)); + RetryPolicyBuilder> policy = RetryPolicy.builder(); + if (BASE_DELAY.compareTo(maxDelay) < 0) { + policy.withBackoff(BASE_DELAY, maxDelay, 2.0); + } else { + // a total duration too short for the pause to grow leaves one pause of the capped size + policy.withDelay(maxDelay); + } + return policy + .handle(IOException.class) + .handleIf((ignored, failure) -> + failure instanceof RetryableStatus status && worthAnotherAttempt(status)) + .withJitter(JITTER) + .withMaxAttempts(-1) + .withMaxDuration(Duration.ofMillis(maxTotalMillis)) + .onRetry(event -> log.warn("Retrying request: {}. Attempt {} failed with {}, within {}ms total", + compiledReq, event.getAttemptCount(), describe(event.getLastException()), maxTotalMillis)) + .build(); + } - if (errorHandler.containsKey(response.code())) { - errorHandler.get(response.code()).accept(bodyAsString(response)); - return Optional.empty(); - } + /** One request/response exchange. Throws {@link RetryableStatus} for an outcome worth repeating. */ + private Optional attempt(Request compiledReq, long remainingMs, + ExecutionContext> context) throws IOException { + try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { + // check response codes against acceptable list + log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); - if (!expectedCodes.contains(response.code())) { - retryOrFail(response, compiledReq, attempts, deadlineNanos, maxTotalMillis); - continue; - } + if (errorHandler.containsKey(response.code())) { + errorHandler.get(response.code()).accept(bodyAsString(response)); + return Optional.empty(); + } - String body = bodyAsString(response); - log.debug("Response body: {}", body); - return Optional.of(body); - } catch (IOException e) { - if (!canRetry(deadlineNanos)) { - throw new MaaSHttpException("Error executing " + compiledReq + giveUpSuffix(attempts.count), e); + if (!expectedCodes.contains(response.code())) { + // read once, without throwing: a body that cannot be read must not turn a permanent + // status into a retry + String errorBody = errorBodyOrPlaceholder(response); + if (isRetryableStatus(response.code(), errorBody)) { + throw new RetryableStatus(response.code(), errorBody); } - attempts.afterTransportError(e); - log.warn("Error execute http request: {}, Retry {}, within {}ms total", - e.getMessage(), attempts.count, maxTotalMillis); - sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); + throw new MaaSHttpException("Unexpected status code " + response.code() + + " for request: " + compiledReq + + giveUpSuffix(context == null ? 0 : context.getAttemptCount()) + + "\n\tResponse body: " + errorBody); } + + String body = bodyAsString(response); + log.debug("Response body: {}", body); + return Optional.of(body); } } - /** - * Handles a status the caller did not expect: waits before the next attempt, or throws when - * the status is terminal. - */ - private void retryOrFail(Response response, Request compiledReq, Attempts attempts, - long deadlineNanos, long maxTotalMillis) { - // read once, without throwing: a body that cannot be read must not turn a permanent - // status into a retry - String errorBody = errorBodyOrPlaceholder(response); - if (!takeRetrySlotFor(response.code(), errorBody, deadlineNanos)) { - throw new MaaSHttpException("Unexpected status code " + response.code() - + " for request: " + compiledReq - + giveUpSuffix(attempts.count) - + "\n\tResponse body: " + errorBody); + /** The {@link #noRetry()} path, and a total duration configured to zero. */ + private Optional attemptOnce(Request compiledReq) { + try { + return attempt(compiledReq, 0, null); + } catch (RetryableStatus e) { + throw new MaaSHttpException("Unexpected status code " + e.code + + " for request: " + compiledReq + "\n\tResponse body: " + e.body); + } catch (IOException e) { + throw new MaaSHttpException("Error executing " + compiledReq, e); } - attempts.afterStatus(response.code(), errorBody); - log.warn("Retryable status code {} for request: {}. Retry {}, within {}ms total", - response.code(), compiledReq, attempts.count, maxTotalMillis); - sleepBackoff(attempts.count, maxTotalMillis, deadlineNanos); } - /** How many attempts went out and what the last one failed with. */ - private static final class Attempts { - private int count; - private Throwable lastFailure; - private String lastStatusAndBody; + private static String describe(Throwable failure) { + return failure instanceof RetryableStatus status ? status.describe() : String.valueOf(failure); + } - /** The total duration bounds retries, not the call: the first attempt always goes out. */ - boolean outOfTime(long remainingMs) { - return count > 0 && remainingMs <= 0; - } + /** + * The usual terminal failure: a call that keeps failing lands on the deadline. Carries what the + * last attempt saw, otherwise the trace says only that a minute went by. + */ + private static MaaSHttpException exhausted(Request req, long maxTotalMillis, String lastAttempt, Throwable cause) { + String message = "Gave up on " + req + ": ran out of its " + maxTotalMillis + + "ms total duration.\n\tLast attempt: " + lastAttempt; + return cause == null ? new MaaSHttpException(message) : new MaaSHttpException(message, cause); + } - void afterStatus(int code, String body) { - count++; - lastFailure = null; - lastStatusAndBody = "status " + code + ", body: " + body; - } + /** A status the caller did not expect, but one worth another attempt. Never leaves this class. */ + private static final class RetryableStatus extends RuntimeException { + private final transient int code; + private final transient String body; - void afterTransportError(IOException e) { - count++; - lastFailure = e; - lastStatusAndBody = null; + RetryableStatus(int code, String body) { + super(null, null, false, false); + this.code = code; + this.body = body; } - String describeLast() { - if (lastStatusAndBody != null) { - return lastStatusAndBody; - } - return lastFailure != null ? lastFailure.toString() : "unknown"; + String describe() { + return "status " + code + ", body: " + body; } } - - /** - * The usual terminal failure: the backoff is clamped to the time left, so a call that keeps - * failing lands exactly on the deadline. Carries what the last attempt saw, otherwise the - * trace says only that a minute went by. - */ - private static MaaSHttpException totalDurationExceeded(Request req, Attempts attempts, long maxTotalMillis) { - String message = "Gave up on " + req + " after " + attempts.count + " retries: ran out of its " - + maxTotalMillis + "ms total duration." - + "\n\tLast attempt: " + attempts.describeLast(); - return attempts.lastFailure != null - ? new MaaSHttpException(message, attempts.lastFailure) - : new MaaSHttpException(message); - } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 685d094085..a7c10f8922 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -289,22 +289,6 @@ void testTotalDurationExceeded_CarriesTheLastFailureAsCause() throws IOException } } - // Delay must grow between attempts and saturate at a quarter of the total duration. - @Test - void testBackoffMillis_GrowsAndSaturatesAtTheCap() { - long[] expectedFor60s = {1_000, 2_000, 4_000, 8_000, 15_000, 15_000}; - for (int attempt = 1; attempt <= expectedFor60s.length; attempt++) { - assertEquals(expectedFor60s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 60_000), - "attempt " + attempt + " of a 60s total duration"); - } - - long[] expectedFor5s = {1_000, 1_250, 1_250}; - for (int attempt = 1; attempt <= expectedFor5s.length; attempt++) { - assertEquals(expectedFor5s[attempt - 1], HttpExecution.cappedDelayMillis(attempt, 5_000), - "attempt " + attempt + " of a 5s total duration"); - } - } - // A tight max total duration must cut the retry loop short well before the attempt count is exhausted. @Test void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServer) { diff --git a/maas-client/pom.xml b/maas-client/pom.xml index c847670231..8473452884 100644 --- a/maas-client/pom.xml +++ b/maas-client/pom.xml @@ -25,6 +25,7 @@ 1.1.3-SNAPSHOT 4.2.1 + 3.3.2 ${project.basedir}/../report-aggregate/target/site/jacoco-aggregate/jacoco.xml @@ -93,6 +94,11 @@ kafka-streams ${kafka.version} + + dev.failsafe + failsafe + ${failsafe.version} + From 7c23c3d147a13ce3c53dc3b99737281a5ca9723f Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 12:00:59 +0400 Subject: [PATCH 16/24] chore: fix tests after rebase onto main --- .../impl/kafka/KafkaMaaSClientWatchBackoffTest.java | 5 ++--- .../maas/client/impl/rabbit/RabbitFailoverTest.java | 9 ++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index f5521af4ef..3d9019e7e0 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -23,7 +23,6 @@ import com.netcracker.cloud.maas.client.impl.Env; import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; import com.netcracker.cloud.maas.client.impl.http.HttpClient; -import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; @@ -108,10 +107,10 @@ void failingWatchPollIsBackedOffInsteadOfHotLooping() { } private static KafkaMaaSClientImpl createKafkaClient(String agentUrl) { - System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, agentUrl); + System.setProperty(Env.PROP_MAAS_AGENT_URL, agentUrl); var httpClient = HttpClient.getMaasClient(() -> "faketoken"); var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); - System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); + System.clearProperty(Env.PROP_MAAS_AGENT_URL); return new KafkaMaaSClientImpl(httpClient, () -> { throw new UnsupportedOperationException("tenant manager is not used in this test"); }, diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java index 048911468c..bf24da0692 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -7,7 +7,6 @@ import com.netcracker.cloud.maas.client.impl.Env; import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; import com.netcracker.cloud.maas.client.impl.http.HttpClient; -import com.netcracker.cloud.security.core.utils.k8s.M2MClientFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -32,7 +31,7 @@ class RabbitFailoverTest { @BeforeEach void reset(ClientAndServer mockServer) { - savedAgentUrl = System.getProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); + savedAgentUrl = System.getProperty(Env.PROP_MAAS_AGENT_URL); mockServer.reset(); mockServer.when(request().withPath("/api-version")) .respond(response().withBody("{\"major\":2, \"minor\": 16}")); @@ -43,9 +42,9 @@ void reset(ClientAndServer mockServer) { @AfterEach void restoreAgentUrl() { if (savedAgentUrl == null) { - System.clearProperty(M2MClientFactory.MAAS_AGENT_URL_PROP); + System.clearProperty(Env.PROP_MAAS_AGENT_URL); } else { - System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, savedAgentUrl); + System.setProperty(Env.PROP_MAAS_AGENT_URL, savedAgentUrl); } } @@ -119,7 +118,7 @@ private static void withFastRetries(Runnable test) { } private static RabbitMaaSClientImpl createRabbitClient(String agentUrl) { - System.setProperty(M2MClientFactory.MAAS_AGENT_URL_PROP, agentUrl); + System.setProperty(Env.PROP_MAAS_AGENT_URL, agentUrl); var httpClient = HttpClient.getMaasClient(() -> "faketoken"); var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); return new RabbitMaaSClientImpl(httpClient, new ApiUrlProvider(serverApiVersion, agentUrl)); From bfb8a45996689a6a66d39e22bd9b23948f499896 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 13:17:24 +0400 Subject: [PATCH 17/24] chore: tests refactoring --- .../cloud/maas/client/api/MaaSException.java | 6 +- .../maas/client/api/MaaSHttpException.java | 6 +- .../cloud/maas/client/impl/Env.java | 8 +- .../maas/client/impl/http/HttpExecution.java | 125 +++++------------- .../impl/kafka/KafkaMaaSClientImpl.java | 40 ++---- .../impl/http/HttpExecutionFailoverTest.java | 86 +++--------- .../impl/rabbit/RabbitFailoverTest.java | 41 ++---- 7 files changed, 80 insertions(+), 232 deletions(-) diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java index d0a16bfcd9..2989f49a5d 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSException.java @@ -2,15 +2,11 @@ public class MaaSException extends RuntimeException { - /** - * Formats the message. Note that a two-argument call whose second argument is a - * {@code Throwable} binds to the constructor below instead, and is not formatted. - */ public MaaSException(String format, Object...args) { super(String.format(format, args)); } - /** For subclasses whose message is already built and must not go through String.format. */ + /** For a message that is already built and must not go through String.format. */ protected MaaSException(String message, Throwable cause) { super(message, cause); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java index 69e1a470a7..00fad80b77 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java @@ -1,10 +1,6 @@ package com.netcracker.cloud.maas.client.api; -/** - * A call to maas-agent that did not succeed: an unexpected status code, or a transport - * failure that outlived the configured total duration. The message is taken as is, unlike - * {@link MaaSException}, because it carries request and response text. - */ +/** A call to maas that did not succeed: an unexpected status code or a transport failure. */ public class MaaSHttpException extends MaaSException { public MaaSHttpException(String message) { diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index b3dc649894..b957728ab6 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -113,12 +113,8 @@ public static Duration httpTimeout() { static final long DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS = 60_000L; /** - * How long one call to maas-agent may take in total, retries included. The only retry - * knob: attempt count and backoff growth are derived from it. The 60s default outlasts - * a database leader switchover. - *

- * Zero disables retries, leaving a single attempt. An unreadable or negative value falls - * back to the default with a warning, rather than failing the call that happens to be first. + * How long one call may take in total, retries included. Zero leaves a single + * attempt; an unreadable or negative value falls back to the default with a warning. */ public static Duration httpRetryMaxTotalDuration() { return Duration.ofMillis( diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 0392146641..412494c372 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netcracker.cloud.maas.client.api.MaaSHttpException; import com.netcracker.cloud.maas.client.impl.Env; -import dev.failsafe.ExecutionContext; import dev.failsafe.Failsafe; import dev.failsafe.FailsafeException; import dev.failsafe.RetryPolicy; @@ -17,6 +16,7 @@ import java.io.IOException; import java.time.Duration; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; @@ -111,29 +111,17 @@ public Optional sendAndReceive(OmnivoreFunction responseDeseri private static final String MAAS_ERROR_CODE = "MAAS-0600"; /** - * Two 4xx are transient here rather than permanent: 405 is how maas-service reports a - * read-only Postgres during a leader switchover, and 401 clears when the token is - * re-supplied on the next attempt. - *

- * The 405 case is gated on the response body, because a plain 405 — a route removed on - * the server, an ingress rejecting the method — is permanent and must fail fast. + * Whether the status is worth another attempt. 405 and 401 are here because maas-service + * reports a read-only database as 405, and a token can expire in flight. */ private static boolean isRetryableStatus(int code, String body) { - if (code >= 500) { - return true; - } - if (code == 429 || code == 401) { + if (code >= 500 || code == 429 || code == 401) { return true; } return code == 405 && isDatabaseUnavailable(body); } - /** - * Recognises the 405 that maas-service returns for PostgreSQL error 25006, mapped from - * {@code DatabaseIsReadonlyError} ("database is in read-only mode") and - * {@code DatabaseIsNotActiveError} ("database is not in 'active' mode"), both declared in - * maas-service and mapped to 405. - */ + /** Tells the 405 of a read-only database apart from a plain one, which is permanent. */ private static boolean isDatabaseUnavailable(String body) { if (body == null || !body.contains(MAAS_ERROR_CODE)) { return false; @@ -143,11 +131,7 @@ private static boolean isDatabaseUnavailable(String body) { || reason.contains("not in 'active' mode"); } - /** - * How many times a single call retries a 401. One is enough: it covers a token that - * expired in flight. A token the supplier still considers valid but the server rejects - * comes back identical on every further attempt. - */ + /** One is enough: further attempts would re-send the same token. */ static final int MAX_AUTH_RETRIES = 1; private int authAttempts = 0; @@ -168,10 +152,7 @@ private static String bodyAsString(Response response) throws IOException { return body == null ? "" : body.string(); } - /** - * Body of a non-2xx response, for the retry decision and the error message. Never throws: - * a body that cannot be read must not turn a permanent status into a retry. - */ + /** Body of a non-2xx response. Never throws: an unreadable body must not become a retry. */ private static String errorBodyOrPlaceholder(Response response) { try { return bodyAsString(response); @@ -181,33 +162,7 @@ private static String errorBodyOrPlaceholder(Response response) { } } - private static String giveUpSuffix(int attempt) { - return attempt == 0 ? "" : ", gave up after " + attempt + " retries"; - } - - /** - * Client for one attempt, bounded by what is left of the total duration. Without it an - * attempt starting just before the deadline still runs for the full - * {@code maas.http.timeout} and the call overruns its total duration. - *

- * Not applied under {@link #noRetry()}: there the caller owns the lifecycle, and the - * watch long poll legitimately runs as long as the total duration itself. - */ - private OkHttpClient clientForAttempt(long remainingMs) { - if (!retryEnabled || remainingMs <= 0) { - // no retries, or a zero total duration: the single attempt keeps the client's own timeouts - return httpClient; - } - // newBuilder shares the connection pool and dispatcher, so this is cheap - return httpClient.newBuilder() - .callTimeout(Duration.ofMillis(remainingMs)) - .build(); - } - - /** - * Backoff, jitter, attempt counting and the overall deadline belong to the retry policy; - * what is left here is what one attempt is and which of its outcomes is worth repeating. - */ + /** Sends the request, retrying under the policy below until it succeeds or runs out of time. */ private Optional sendAndReceive() { Request compiledReq = req.build(); log.debug("Send request: {}", compiledReq); @@ -219,16 +174,17 @@ private Optional sendAndReceive() { authAttempts = 0; try { return Failsafe.with(retryPolicy(compiledReq, maxTotalMillis)).get(context -> - attempt(compiledReq, maxTotalMillis - context.getElapsedTime().toMillis(), context)); - } catch (RetryableStatus e) { - throw exhausted(compiledReq, maxTotalMillis, e.describe(), null); - } catch (FailsafeException e) { - if (e.getCause() instanceof InterruptedException interrupted) { + attempt(compiledReq, maxTotalMillis - context.getElapsedTime().toMillis(), true)); + } catch (RetryableStatus | FailsafeException e) { + Throwable last = e instanceof FailsafeException failsafe ? failsafe.getCause() : null; + if (last instanceof InterruptedException interrupted) { // Failsafe restores the flag; the call still has to abort rather than retry Thread.currentThread().interrupt(); throw new MaaSHttpException("Interrupted while waiting to retry maas-agent request", interrupted); } - throw exhausted(compiledReq, maxTotalMillis, String.valueOf(e.getCause()), e.getCause()); + throw new MaaSHttpException("Gave up on " + compiledReq + ": ran out of its " + + maxTotalMillis + "ms total duration.\n\tLast attempt: " + + (last != null ? last : e), last); } } @@ -238,8 +194,7 @@ private RetryPolicy> retryPolicy(Request compiledReq, long maxT if (BASE_DELAY.compareTo(maxDelay) < 0) { policy.withBackoff(BASE_DELAY, maxDelay, 2.0); } else { - // a total duration too short for the pause to grow leaves one pause of the capped size - policy.withDelay(maxDelay); + policy.withDelay(maxDelay); // too short for the pause to grow } return policy .handle(IOException.class) @@ -249,14 +204,18 @@ failure instanceof RetryableStatus status && worthAnotherAttempt(status)) .withMaxAttempts(-1) .withMaxDuration(Duration.ofMillis(maxTotalMillis)) .onRetry(event -> log.warn("Retrying request: {}. Attempt {} failed with {}, within {}ms total", - compiledReq, event.getAttemptCount(), describe(event.getLastException()), maxTotalMillis)) + compiledReq, event.getAttemptCount(), event.getLastException(), maxTotalMillis)) .build(); } /** One request/response exchange. Throws {@link RetryableStatus} for an outcome worth repeating. */ - private Optional attempt(Request compiledReq, long remainingMs, - ExecutionContext> context) throws IOException { - try (Response response = clientForAttempt(remainingMs).newCall(compiledReq).execute()) { + private Optional attempt(Request compiledReq, long remainingMs, boolean retrying) throws IOException { + Call call = httpClient.newCall(compiledReq); + if (remainingMs > 0) { + // an attempt starting near the deadline must not overrun the total duration + call.timeout().timeout(remainingMs, TimeUnit.MILLISECONDS); + } + try (Response response = call.execute()) { // check response codes against acceptable list log.debug("Received status code: {}, expected codes: {}", response.code(), expectedCodes); @@ -266,16 +225,12 @@ private Optional attempt(Request compiledReq, long remainingMs, } if (!expectedCodes.contains(response.code())) { - // read once, without throwing: a body that cannot be read must not turn a permanent - // status into a retry String errorBody = errorBodyOrPlaceholder(response); - if (isRetryableStatus(response.code(), errorBody)) { + if (retrying && isRetryableStatus(response.code(), errorBody)) { throw new RetryableStatus(response.code(), errorBody); } throw new MaaSHttpException("Unexpected status code " + response.code() - + " for request: " + compiledReq - + giveUpSuffix(context == null ? 0 : context.getAttemptCount()) - + "\n\tResponse body: " + errorBody); + + " for request: " + compiledReq + "\n\tResponse body: " + errorBody); } String body = bodyAsString(response); @@ -287,42 +242,24 @@ private Optional attempt(Request compiledReq, long remainingMs, /** The {@link #noRetry()} path, and a total duration configured to zero. */ private Optional attemptOnce(Request compiledReq) { try { - return attempt(compiledReq, 0, null); - } catch (RetryableStatus e) { - throw new MaaSHttpException("Unexpected status code " + e.code - + " for request: " + compiledReq + "\n\tResponse body: " + e.body); + return attempt(compiledReq, 0, false); } catch (IOException e) { throw new MaaSHttpException("Error executing " + compiledReq, e); } } - private static String describe(Throwable failure) { - return failure instanceof RetryableStatus status ? status.describe() : String.valueOf(failure); - } - - /** - * The usual terminal failure: a call that keeps failing lands on the deadline. Carries what the - * last attempt saw, otherwise the trace says only that a minute went by. - */ - private static MaaSHttpException exhausted(Request req, long maxTotalMillis, String lastAttempt, Throwable cause) { - String message = "Gave up on " + req + ": ran out of its " + maxTotalMillis - + "ms total duration.\n\tLast attempt: " + lastAttempt; - return cause == null ? new MaaSHttpException(message) : new MaaSHttpException(message, cause); - } - /** A status the caller did not expect, but one worth another attempt. Never leaves this class. */ private static final class RetryableStatus extends RuntimeException { private final transient int code; - private final transient String body; RetryableStatus(int code, String body) { - super(null, null, false, false); + super("status " + code + ", body: " + body, null, false, false); this.code = code; - this.body = body; } - String describe() { - return "status " + code + ", body: " + body; + @Override + public String toString() { + return getMessage(); } } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 565bccebc0..f2fbb78024 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -42,21 +42,15 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { private final ApiUrlProvider apiProvider; /** - * How long maas-service is asked to hold a watch poll open. It must stay below the client's - * own read timeout, otherwise every quiet poll dies locally instead of returning an empty - * 200 — which counts as a failure and walks the backoff up to its cap, delaying the next - * real topic-create event. maas-service caps the window at 120s in any case. + * How long maas-service is asked to hold a watch poll open. Stays below the read timeout, + * otherwise a quiet poll dies locally instead of returning an empty 200. */ private final Duration watchTimeout = watchTimeout(Env.httpTimeout()); /** Largest gap left between the watch window and the read timeout. */ private static final long MAX_WATCH_MARGIN_SECONDS = 5; - /** - * The margin is clamped rather than subtracted outright, so that a small read timeout - * narrows the window instead of pushing it past the timeout. Whole seconds, because that - * is how the window travels in the query string. - */ + /** The margin is clamped, so a small read timeout narrows the window instead of inverting it. */ static Duration watchTimeout(Duration httpTimeout) { long timeoutSeconds = httpTimeout.getSeconds(); long marginSeconds = Math.min(MAX_WATCH_MARGIN_SECONDS, timeoutSeconds / 2); @@ -69,11 +63,8 @@ static Duration watchTimeout(Duration httpTimeout) { private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); private volatile boolean closed = false; /** - * Monitor for parking the watch thread while there is nothing to watch. - *

- * Not the thread itself: {@link Thread#join()} waits on that same monitor and the JVM - * notifies it when the thread ends, so a notification meant for the watch loop can be - * consumed by a {@code join()} in {@link #close()} and the loop never wakes up. + * Monitor for parking the watch thread. Not the thread itself: {@link Thread#join()} waits on + * that monitor too, and would steal the notification meant for the loop. */ private final Object watchLock = new Object(); private final Lazy watchThread = new Lazy<>(() -> { @@ -123,8 +114,7 @@ public boolean deleteTopic(Classifier classifier) { .orElse(null); if (resp == null) { - // empty body: nothing was reported as deleted - return false; + return false; // empty body } if (!resp.getFailedToDelete().isEmpty()) { throw new MaaSException("Error delete topic by classifier: %s. Error: %s", classifier, resp.getFailedToDelete().get(0).getMessage()); @@ -212,20 +202,16 @@ private void deliver(List found) { continue; } for (Consumer callback : callbacks) { - notifyCallback(addr, callback); + try { + log.info("Topic create event for {} received, execute callback {}", addr.getClassifier(), callback); + callback.accept(new TopicAddressImpl(addr)); + } catch (Exception e) { + log.error("Error execute callback {}", callback, e); + } } } } - private void notifyCallback(TopicInfo addr, Consumer callback) { - try { - log.info("Topic create event for {} received, execute callback {}", addr.getClassifier(), callback); - callback.accept(new TopicAddressImpl(addr)); - } catch (Exception e) { - log.error("Error execute callback {}", callback, e); - } - } - /** * Parks the thread while no topic is being watched. * @@ -332,7 +318,7 @@ public List search(SearchCriteria criteria) { .post(criteria) .expect(HTTP_OK) .sendAndReceive(typeRef) - .get() + .orElseGet(Collections::emptyList) .stream() .map(TopicAddressImpl::new) .collect(Collectors.toList()); diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index a7c10f8922..ad28c40c02 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -42,65 +42,37 @@ class HttpExecutionFailoverTest { private static final String PATH = "/api/v1/kafka/topic"; - @Test - void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.exactly(2)) - .respond(response().withStatusCode(405) - .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}")); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(200).withBody("\"ok\"")); - - withFastRetries(() -> { - Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); - assertTrue(body.isPresent()); - assertEquals("ok", body.get()); - }); - - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); - } - - @Test - void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.exactly(2)) - .respond(response().withStatusCode(500) - .withBody("{\"error\":\"error proxying request: connection refused\"}")); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(200).withBody("\"ok\"")); - - withFastRetries(() -> { - Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); - assertTrue(body.isPresent()); - assertEquals("ok", body.get()); - }); - - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(3)); + static Stream retryableResponses() { + return Stream.of( + arguments("a read-only database", 405, + "{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}", 2), + arguments("maas-agent unable to reach maas-service", 500, + "{\"error\":\"error proxying request: connection refused\"}", 2), + arguments("throttling", 429, "{\"error\":\"slow down\"}", 1), + // an expired token clears on the next attempt, because the supplier is called again + arguments("an expired token", 401, "{\"error\":\"unauthorized\"}", 1) + ); } - /** An expired token clears on the next attempt, because the supplier is called again. */ - @Test - void testFailover_401ThenSuccess(ClientAndServer mockServer) { + @ParameterizedTest(name = "{0} is retried") + @MethodSource("retryableResponses") + void testFailover_RetryableResponseSucceedsOnRetry(String description, int status, String body, int failures, + ClientAndServer mockServer) { mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.exactly(1)) - .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); + mockServer.when(request().withPath(PATH), Times.exactly(failures)) + .respond(response().withStatusCode(status).withBody(body)); mockServer.when(request().withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(200).withBody("\"ok\"")); - withFastRetries(() -> { - Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); - assertTrue(body.isPresent()); - assertEquals("ok", body.get()); - }); + withFastRetries(() -> + assertEquals("ok", execution(mockServer).expect(200).sendAndReceive(String.class).orElseThrow())); - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(2)); + mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(failures + 1)); } /** - * A 401 that keeps coming back means the supplier is handing out a token the server - * rejects, and it has no way of being told so. Further attempts resend the same token, - * so retrying it is deliberately capped tighter than the total duration: a wrong secret must - * fail fast instead of hanging for the whole minute. + * A 401 that keeps coming back means the supplier hands out a token the server rejects and + * cannot be told so, hence the tighter cap: a wrong secret must fail fast. */ @Test void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { @@ -170,22 +142,6 @@ void testMaxTotalDuration_BoundsAHangingAttempt() throws IOException { } } - @Test - void testFailover_429Retried(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.exactly(1)) - .respond(response().withStatusCode(429).withBody("{\"error\":\"slow down\"}")); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(200).withBody("\"ok\"")); - - withFastRetries(() -> { - Optional body = execution(mockServer).expect(200).sendAndReceive(String.class); - assertEquals("ok", body.orElseThrow()); - }); - - mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(2)); - } - /** The watch long poll owns its own loop, so its execution must send the request exactly once. */ @Test void testNoRetry_SendsExactlyOneAttempt(ClientAndServer mockServer) { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java index bf24da0692..d0dae5b15a 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -11,6 +11,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.mockserver.integration.ClientAndServer; import org.mockserver.junit.jupiter.MockServerExtension; import org.mockserver.matchers.Times; @@ -48,35 +50,15 @@ void restoreAgentUrl() { } } - @Test - void testFailover_405TwiceThenSuccess(ClientAndServer mockServer) { - mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) - .respond(response().withStatusCode(405) - .withBody("{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}")); - mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(200).withBody(""" - { - "cnn": "ampq://rabbit-cluster:4321/maas.core-dev.123456", - "username": "testuser", - "password": "plain:testpassword" - } - """)); - - withProp(Env.PROP_NAMESPACE, "core-dev", () -> - withFastRetries(() -> { - RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); - VHost vhost = client.getOrCreateVirtualHost(new Classifier("commands")); - assertNotNull(vhost); - })); - - mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(3)); - } - - @Test - void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { + @ParameterizedTest(name = "{0} is retried on the vhost path") + @CsvSource(delimiter = '|', textBlock = """ + a read-only database | 405 | {"code":"MAAS-0600","reason":"database is in read-only mode"} + an unreachable agent | 500 | {"error":"error proxying request: connection refused"} + """) + void testFailover_RetryableResponseSucceedsOnRetry(String description, int status, String body, + ClientAndServer mockServer) { mockServer.when(request().withMethod("POST").withPath(PATH), Times.exactly(2)) - .respond(response().withStatusCode(500) - .withBody("{\"error\":\"error proxying request: connection refused\"}")); + .respond(response().withStatusCode(status).withBody(body)); mockServer.when(request().withMethod("POST").withPath(PATH), Times.unlimited()) .respond(response().withStatusCode(200).withBody(""" { @@ -89,8 +71,7 @@ void testFailover_500TwiceThenSuccess(ClientAndServer mockServer) { withProp(Env.PROP_NAMESPACE, "core-dev", () -> withFastRetries(() -> { RabbitMaaSClientImpl client = createRabbitClient("http://localhost:" + mockServer.getPort()); - VHost vhost = client.getOrCreateVirtualHost(new Classifier("commands")); - assertNotNull(vhost); + assertNotNull(client.getOrCreateVirtualHost(new Classifier("commands"))); })); mockServer.verify(request().withMethod("POST").withPath(PATH), VerificationTimes.exactly(3)); From 895b45687bf8c8ce109a1211a8b4ed52ef6eb7f3 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 13:53:33 +0400 Subject: [PATCH 18/24] chore: sonar warnings fix --- .../impl/kafka/KafkaMaaSClientImpl.java | 21 ++++++++++++------- .../impl/http/HttpExecutionFailoverTest.java | 1 - .../impl/rabbit/RabbitFailoverTest.java | 1 - 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index f2fbb78024..775026385d 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -170,8 +170,8 @@ private boolean pollWhileThereIsSomethingToWatch() { } failures++; log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); - if (!sleepWatchBackoff(failures)) { - return false; // interrupted while backing off + if (!awaitWatchBackoff(failures)) { + return false; // closed or interrupted while backing off } continue; // nothing was received, nothing to deliver } @@ -235,17 +235,24 @@ private boolean parkUntilThereIsSomethingToWatch() { } /** - * Linear, capped backoff between failed watch polls, reset on every success. + * Linear, capped backoff between failed watch polls, reset on every success. Waits on the + * watch monitor rather than sleeping, so close() cuts the wait short. * - * @return false if the thread was interrupted while waiting, meaning the caller should stop + * @return false if the client was closed or the thread interrupted, meaning the caller stops */ - private static boolean sleepWatchBackoff(int failures) { + private boolean awaitWatchBackoff(int failures) { long delayMillis = Math.min( failures * WATCH_RETRY_INTERVAL.toMillis(), WATCH_MAX_RETRY_INTERVAL.toMillis()); + long deadline = System.currentTimeMillis() + delayMillis; try { - Thread.sleep(delayMillis); - return true; + synchronized (watchLock) { + // waits out the whole pause: only close() ends it early, a new watch does not + for (long left = delayMillis; !closed && left > 0; left = deadline - System.currentTimeMillis()) { + watchLock.wait(left); + } + } + return !closed; } catch (InterruptedException e) { Thread.currentThread().interrupt(); return false; diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index ad28c40c02..afe3c1a52b 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -20,7 +20,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java index d0dae5b15a..6dcbbe2cb4 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/rabbit/RabbitFailoverTest.java @@ -2,7 +2,6 @@ import com.netcracker.cloud.maas.client.api.Classifier; import com.netcracker.cloud.maas.client.api.MaaSHttpException; -import com.netcracker.cloud.maas.client.api.rabbit.VHost; import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; import com.netcracker.cloud.maas.client.impl.Env; import com.netcracker.cloud.maas.client.impl.apiversion.ServerApiVersion; From 6ea4c28ed4d48fac79e1e89f58e19b7c6cc34281 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 16:08:30 +0400 Subject: [PATCH 19/24] fix: remove 401 from reasons for retry + review findings --- maas-client/CHANGELOG.md | 15 ++-- maas-client/README.md | 6 +- .../maas/client/api/MaaSHttpException.java | 4 +- .../maas/client/impl/ApiUrlProvider.java | 2 +- .../cloud/maas/client/impl/Env.java | 8 ++- .../maas/client/impl/http/HttpExecution.java | 36 ++++------ .../impl/kafka/KafkaMaaSClientImpl.java | 69 ++++++++++--------- .../cloud/maas/client/impl/EnvTest.java | 25 +++++++ .../impl/http/HttpExecutionFailoverTest.java | 48 ++++++------- .../impl/kafka/KafkaMaaSClientImplTest.java | 16 +++++ .../KafkaMaaSClientWatchBackoffTest.java | 37 ++++++---- 11 files changed, 156 insertions(+), 110 deletions(-) diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index ca5ebb86ea..60c5783bf4 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -3,10 +3,15 @@ ## Unreleased * `Features` - HTTP calls to maas-agent are now retried on retryable status codes, not only on `IOException`. - Retryable: 5xx, 429, **405** (only when the body carries a maas-service error) and **401** - (once). See "Retry behaviour and configuration" in README for why the two 4xx codes are - included — without them the client does not survive a Postgres leader switchover. - - Backoff is exponential with jitter instead of a fixed 1s delay. + Retryable: 5xx, 429 and **405**, the last one only when the body carries a maas-service error. + See "Retry behaviour and configuration" in README for why a 4xx is included — without it the + client does not survive a Postgres leader switchover. 401 is not retried: the token source + refreshes on its own schedule, so a retry would re-send the same token. + - Backoff is exponential with jitter instead of a fixed 1s delay, implemented with a new runtime + dependency, `dev.failsafe:failsafe`. It has no transitive dependencies, but services with + dependency convergence rules will see it appear. + - `deleteTopic` is deliberately not retried: it is not idempotent, and a lost response after a + completed delete would make the next attempt report the topic as still present. - New configuration: `maas.http.retry.max-total-duration-ms` (`60s` by default) — a single setting bounding the whole call. The attempt count and the backoff growth are derived from it, so there are no separate knobs to keep consistent. Every attempt is bounded by what is @@ -17,7 +22,7 @@ down maas-agent is no longer polled in a hot loop. * `Behaviour changes` - **A call that fails with a retryable status now takes longer before failing.** Previously an - unexpected 5xx/405/401 threw immediately; it is now retried within the configured limits. + unexpected 5xx or 405 threw immediately; it is now retried within the configured limits. - Interrupting a thread during a retry wait now restores the interrupt flag and aborts the loop, instead of swallowing `InterruptedException`. - `KafkaMaaSClient.watchTopicCreate` throws `IllegalStateException` after `close()`, instead of diff --git a/maas-client/README.md b/maas-client/README.md index a075de24bb..c48faa5d00 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -96,11 +96,11 @@ Which responses are retried: | 5xx | yes | includes the `500` maas-agent returns when it cannot reach maas-service at all | | 429 | yes | throttling | | **405** | **only with a maas-service error body** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a switchover arrives as `405`, not as `5xx`. A plain `405` — a route removed on the server, an ingress rejecting the method — is permanent and fails fast | -| **401** | **once** | covers a token that expired in flight. Further attempts re-send the same token, since the supplier cannot be told it was rejected | +| 401 | no | `CachingTokenSource` refreshes on its own polling interval, so a retry within the backoff reads the same token, and `M2MInterceptor` has already made its own 401 round trip by then | | other 4xx | no | permanent client errors, failed on the first attempt | -The two 4xx entries are deliberate: the usual "retry 5xx, fail fast on 4xx" rule -does not survive a database leader switchover here. +The 405 entry is deliberate: the usual "retry 5xx, fail fast on 4xx" rule does not +survive a database leader switchover here. ## Kafka client usage example All MaaS operations for Kafka is collected in [KafkaMaaSClient](https://github.com/Netcracker/qubership-maas-client/blob/main/client/src/main/java/com/netcracker/cloud/maas/client/api/kafka/KafkaMaaSClient.java). To obtain *new* instance of MaaS Kafka client just call: diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java index 00fad80b77..09c6525fb2 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/api/MaaSHttpException.java @@ -3,8 +3,8 @@ /** A call to maas that did not succeed: an unexpected status code or a transport failure. */ public class MaaSHttpException extends MaaSException { - public MaaSHttpException(String message) { - super(message, (Throwable) null); + public static MaaSHttpException of(String message) { + return new MaaSHttpException(message, null); } public MaaSHttpException(String message, Throwable cause) { diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java index c14809c8a6..6299fff692 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java @@ -37,7 +37,7 @@ public String getKafkaTopicSearchUrl() { } public String getKafkaTopicWatchCreateUrl(Duration timeout) { - return String.format("%s/api/v2/kafka/topic/watch-create?timeout=%ds", maasAgentUrl, timeout.getSeconds()); + return String.format("%s/api/v2/kafka/topic/watch-create?timeout=%dms", maasAgentUrl, timeout.toMillis()); } public String getKafkaTopicGetByClassifierUrl() { diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java index b957728ab6..e61cc46231 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/Env.java @@ -233,11 +233,13 @@ private static Optional microProfileConfigOptional(String key) { Object config = getConfig.invoke(null); Method getOptionalValue = config.getClass().getMethod("getOptionalValue", String.class, Class.class); return (Optional) getOptionalValue.invoke(config, key, String.class); - } catch (ClassNotFoundException | NoClassDefFoundError e) { + } catch (ClassNotFoundException e) { // MicroProfile Config is an optional dependency return Optional.empty(); - } catch (Exception e) { - log.trace("MicroProfile Config lookup failed for '{}'", key, e); + } catch (Throwable e) { + // Throwable, not Exception: a broken config provider fails class initialisation with + // an Error, and the caller must still fall back to system properties and environment + log.trace("MicroProfile Config not available or lookup failed for '{}'", key, e); return Optional.empty(); } } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 412494c372..78f93e8435 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -111,11 +111,12 @@ public Optional sendAndReceive(OmnivoreFunction responseDeseri private static final String MAAS_ERROR_CODE = "MAAS-0600"; /** - * Whether the status is worth another attempt. 405 and 401 are here because maas-service - * reports a read-only database as 405, and a token can expire in flight. + * Whether the status is worth another attempt. 405 is here because maas-service reports a + * read-only database that way; 401 is not, because the token source refreshes on its own + * schedule and M2MInterceptor has already retried by the time we see one. */ private static boolean isRetryableStatus(int code, String body) { - if (code >= 500 || code == 429 || code == 401) { + if (code >= 500 || code == 429) { return true; } return code == 405 && isDatabaseUnavailable(body); @@ -131,15 +132,8 @@ private static boolean isDatabaseUnavailable(String body) { || reason.contains("not in 'active' mode"); } - /** One is enough: further attempts would re-send the same token. */ - static final int MAX_AUTH_RETRIES = 1; - - private int authAttempts = 0; - - /** Asked once per failed attempt, so the 401 is counted here. */ - private boolean worthAnotherAttempt(RetryableStatus status) { - return status.code != 401 || ++authAttempts <= MAX_AUTH_RETRIES; - } + /** Keeps the client's own timeouts for an attempt that is not bounded by a total duration. */ + private static final long NO_CLAMP = -1; /** First backoff pause, and the fraction of the total duration a single pause may reach. */ private static final Duration BASE_DELAY = Duration.ofSeconds(1); @@ -171,7 +165,6 @@ private Optional sendAndReceive() { if (!retryEnabled || maxTotalMillis <= 0) { return attemptOnce(compiledReq); } - authAttempts = 0; try { return Failsafe.with(retryPolicy(compiledReq, maxTotalMillis)).get(context -> attempt(compiledReq, maxTotalMillis - context.getElapsedTime().toMillis(), true)); @@ -197,9 +190,7 @@ private RetryPolicy> retryPolicy(Request compiledReq, long maxT policy.withDelay(maxDelay); // too short for the pause to grow } return policy - .handle(IOException.class) - .handleIf((ignored, failure) -> - failure instanceof RetryableStatus status && worthAnotherAttempt(status)) + .handle(IOException.class, RetryableStatus.class) .withJitter(JITTER) .withMaxAttempts(-1) .withMaxDuration(Duration.ofMillis(maxTotalMillis)) @@ -211,9 +202,10 @@ failure instanceof RetryableStatus status && worthAnotherAttempt(status)) /** One request/response exchange. Throws {@link RetryableStatus} for an outcome worth repeating. */ private Optional attempt(Request compiledReq, long remainingMs, boolean retrying) throws IOException { Call call = httpClient.newCall(compiledReq); - if (remainingMs > 0) { - // an attempt starting near the deadline must not overrun the total duration - call.timeout().timeout(remainingMs, TimeUnit.MILLISECONDS); + if (remainingMs >= 0) { + // an attempt starting near the deadline must not overrun the total duration; Failsafe + // truncates the last backoff to land on it, so the last attempt gets the 1ms floor + call.timeout().timeout(Math.max(1, remainingMs), TimeUnit.MILLISECONDS); } try (Response response = call.execute()) { // check response codes against acceptable list @@ -229,7 +221,7 @@ private Optional attempt(Request compiledReq, long remainingMs, boolean if (retrying && isRetryableStatus(response.code(), errorBody)) { throw new RetryableStatus(response.code(), errorBody); } - throw new MaaSHttpException("Unexpected status code " + response.code() + throw MaaSHttpException.of("Unexpected status code " + response.code() + " for request: " + compiledReq + "\n\tResponse body: " + errorBody); } @@ -242,7 +234,7 @@ private Optional attempt(Request compiledReq, long remainingMs, boolean /** The {@link #noRetry()} path, and a total duration configured to zero. */ private Optional attemptOnce(Request compiledReq) { try { - return attempt(compiledReq, 0, false); + return attempt(compiledReq, NO_CLAMP, false); } catch (IOException e) { throw new MaaSHttpException("Error executing " + compiledReq, e); } @@ -250,11 +242,9 @@ private Optional attemptOnce(Request compiledReq) { /** A status the caller did not expect, but one worth another attempt. Never leaves this class. */ private static final class RetryableStatus extends RuntimeException { - private final transient int code; RetryableStatus(int code, String body) { super("status " + code + ", body: " + body, null, false, false); - this.code = code; } @Override diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 775026385d..ca97454cf6 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -8,9 +8,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -48,25 +52,26 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { private final Duration watchTimeout = watchTimeout(Env.httpTimeout()); /** Largest gap left between the watch window and the read timeout. */ - private static final long MAX_WATCH_MARGIN_SECONDS = 5; + private static final Duration MAX_WATCH_MARGIN = Duration.ofSeconds(5); - /** The margin is clamped, so a small read timeout narrows the window instead of inverting it. */ + /** Half the read timeout at most, so even a one-second timeout keeps the window under it. */ static Duration watchTimeout(Duration httpTimeout) { - long timeoutSeconds = httpTimeout.getSeconds(); - long marginSeconds = Math.min(MAX_WATCH_MARGIN_SECONDS, timeoutSeconds / 2); - return Duration.ofSeconds(Math.max(1, timeoutSeconds - marginSeconds)); + Duration margin = httpTimeout.dividedBy(2); + return httpTimeout.minus(margin.compareTo(MAX_WATCH_MARGIN) < 0 ? margin : MAX_WATCH_MARGIN); } private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); private static final Duration WATCH_MAX_RETRY_INTERVAL = Duration.ofSeconds(30); + + /** Grows by one interval per consecutive failure, up to the cap. */ + static long watchBackoffMillis(int failures) { + return Math.min(failures * WATCH_RETRY_INTERVAL.toMillis(), WATCH_MAX_RETRY_INTERVAL.toMillis()); + } // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); private volatile boolean closed = false; - /** - * Monitor for parking the watch thread. Not the thread itself: {@link Thread#join()} waits on - * that monitor too, and would steal the notification meant for the loop. - */ - private final Object watchLock = new Object(); + + private final Semaphore watchSignal = new Semaphore(0); private final Lazy watchThread = new Lazy<>(() -> { Thread exec = new Thread(this::watchTenantCreateTopics, "watchTopicCreate"); exec.setDaemon(true); @@ -110,6 +115,7 @@ public boolean deleteTopic(Classifier classifier) { TopicDeleteResponse resp = httpClient.request(apiProvider.getKafkaTopicUrl(null)) .delete(new TopicDeleteRequest(classifier)) .expect(HTTP_OK) + .noRetry() .sendAndReceive(TopicDeleteResponse.class) .orElse(null); @@ -165,7 +171,9 @@ private boolean pollWhileThereIsSomethingToWatch() { return false; // shutting down, not a failure worth reporting } if (Thread.currentThread().isInterrupted()) { - log.warn("Watch thread interrupted without close(), stopping to watch {}", url, e); + log.error("Watch thread interrupted without close(). Topic create callbacks for {} " + + "will no longer fire; recreate the client to resume watching", + topicCreateListeners.keySet(), e); return false; } failures++; @@ -185,8 +193,12 @@ private boolean pollWhileThereIsSomethingToWatch() { /** One long poll for topics created since the previous call. */ private List poll(String url) { + Set watched; + synchronized (topicCreateListeners) { + watched = new HashSet<>(topicCreateListeners.keySet()); + } return httpClient.request(url) - .post(topicCreateListeners.keySet()) + .post(watched) .expect(200) .noRetry() .sendAndReceive(TOPIC_LIST) @@ -220,12 +232,7 @@ private void deliver(List found) { private boolean parkUntilThereIsSomethingToWatch() { try { log.info("Nothing to watch, sleep thread."); - synchronized (watchLock) { - // guarded wait: a bare wait() would also return on a spurious wakeup - while (!closed && topicCreateListeners.isEmpty()) { - watchLock.wait(); - } - } + watchSignal.acquire(); log.info("Woke up!"); return true; } catch (InterruptedException e) { @@ -235,21 +242,19 @@ private boolean parkUntilThereIsSomethingToWatch() { } /** - * Linear, capped backoff between failed watch polls, reset on every success. Waits on the - * watch monitor rather than sleeping, so close() cuts the wait short. + * Linear, capped backoff between failed watch polls, reset on every success. A permit released + * by close() ends the pause early; one released by a new watch does not shorten it, because + * the permit is put back. * * @return false if the client was closed or the thread interrupted, meaning the caller stops */ private boolean awaitWatchBackoff(int failures) { - long delayMillis = Math.min( - failures * WATCH_RETRY_INTERVAL.toMillis(), - WATCH_MAX_RETRY_INTERVAL.toMillis()); - long deadline = System.currentTimeMillis() + delayMillis; + long deadline = System.currentTimeMillis() + watchBackoffMillis(failures); try { - synchronized (watchLock) { - // waits out the whole pause: only close() ends it early, a new watch does not - for (long left = delayMillis; !closed && left > 0; left = deadline - System.currentTimeMillis()) { - watchLock.wait(left); + for (long left = watchBackoffMillis(failures); !closed && left > 0; + left = deadline - System.currentTimeMillis()) { + if (watchSignal.tryAcquire(left, TimeUnit.MILLISECONDS) && !closed) { + watchSignal.release(); // a new watch, not a close: keep the permit for the park } } return !closed; @@ -271,9 +276,7 @@ public void watchTopicCreate(String name, Consumer callback) { log.info("Add watch for topic by: {}, callback: {}", name, callback); topicCreateListeners.computeIfAbsent(new Classifier(name), k -> Collections.synchronizedList(new ArrayList<>())).add(callback); watchThread.get(); // start the thread if this is the first watch - synchronized (watchLock) { - watchLock.notifyAll(); - } + watchSignal.release(); } @Override @@ -334,9 +337,7 @@ public List search(SearchCriteria criteria) { @Override public void close() { closed = true; - synchronized (watchLock) { - watchLock.notifyAll(); // release the watch thread if it is parked - } + watchSignal.release(); // release the watch thread if it is parked or backing off if (watchThread.isInitialized()) { watchThread.get().interrupt(); try { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/EnvTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/EnvTest.java index 08a85415b1..32d3085710 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/EnvTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/EnvTest.java @@ -156,4 +156,29 @@ void testMicroserviceName() throws Exception { .execute(Env::microserviceName); assertEquals("abc", value); } + + @Test + void testHttpRetryMaxTotalDurationDefault() { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, null, () -> + assertEquals(Env.DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS, + Env.httpRetryMaxTotalDuration().toMillis())); + } + + @Test + void testHttpRetryMaxTotalDurationIsRead() { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, " 1500 ", () -> + assertEquals(1500, Env.httpRetryMaxTotalDuration().toMillis(), "surrounding spaces are tolerated")); + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "0", () -> + assertEquals(0, Env.httpRetryMaxTotalDuration().toMillis(), "zero disables retries")); + } + + /** An unusable value must not fail the call that happens to be first; it falls back and warns. */ + @Test + void testHttpRetryMaxTotalDurationFallsBackOnUnusableValue() { + for (String raw : new String[]{"soon", "", "-1"}) { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, raw, () -> + assertEquals(Env.DEFAULT_HTTP_RETRY_MAX_TOTAL_DURATION_MS, + Env.httpRetryMaxTotalDuration().toMillis(), "unusable value: '" + raw + "'")); + } + } } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index afe3c1a52b..768e885e36 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -47,9 +48,7 @@ static Stream retryableResponses() { "{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}", 2), arguments("maas-agent unable to reach maas-service", 500, "{\"error\":\"error proxying request: connection refused\"}", 2), - arguments("throttling", 429, "{\"error\":\"slow down\"}", 1), - // an expired token clears on the next attempt, because the supplier is called again - arguments("an expired token", 401, "{\"error\":\"unauthorized\"}", 1) + arguments("throttling", 429, "{\"error\":\"slow down\"}", 1) ); } @@ -69,26 +68,13 @@ void testFailover_RetryableResponseSucceedsOnRetry(String description, int statu mockServer.verify(request().withPath(PATH), VerificationTimes.exactly(failures + 1)); } - /** - * A 401 that keeps coming back means the supplier hands out a token the server rejects and - * cannot be told so, hence the tighter cap: a wrong secret must fail fast. - */ - @Test - void testFailover_401GivesUpAfterMaxAuthRetries(ClientAndServer mockServer) { - mockServer.reset(); - mockServer.when(request().withPath(PATH), Times.unlimited()) - .respond(response().withStatusCode(401).withBody("{\"error\":\"unauthorized\"}")); - - withFastRetries(() -> assertMessageContains("401", execution(mockServer).expect(200))); - - mockServer.verify(request().withPath(PATH), - VerificationTimes.exactly(HttpExecution.MAX_AUTH_RETRIES + 1)); - } - /** Responses that are permanent, so the call must fail on its first attempt. */ static Stream permanentResponses() { return Stream.of( arguments("a plain client error", 400, "{\"error\":\"bad request\"}"), + // the token source refreshes on its own schedule and M2MInterceptor has already + // retried, so repeating the call here only sends the same token again + arguments("a rejected token", 401, "{\"error\":\"unauthorized\"}"), // 405 is transient only for a read-only database; a route removed on the server // or an ingress rejecting the method is not arguments("405 without a maas-service envelope", 405, "Method Not Allowed"), @@ -207,6 +193,7 @@ void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { worker.start(); assertTrue(firstAttemptFailed.await(10, TimeUnit.SECONDS), "the first attempt never reached the server"); + awaitState(worker, Thread.State.TIMED_WAITING); worker.interrupt(); worker.join(10_000); @@ -227,7 +214,11 @@ void testInterrupt_RestoresFlagAndAbortsRetryLoop() throws IOException { @Test void testTotalDurationExceeded_CarriesTheLastFailureAsCause() throws IOException { try (ServerSocket rudeServer = new ServerSocket(0)) { - startAcceptor(rudeServer, Socket::close); + AtomicInteger attempts = new AtomicInteger(); + startAcceptor(rudeServer, socket -> { + attempts.incrementAndGet(); + socket.close(); + }); withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1500", () -> { Request.Builder req = new Request.Builder() @@ -240,6 +231,9 @@ void testTotalDurationExceeded_CarriesTheLastFailureAsCause() throws IOException assertTrue(e.getMessage().contains("ran out of its"), "unexpected message: " + e.getMessage()); assertInstanceOf(IOException.class, e.getCause(), "the transport failure that consumed the time must be the cause"); + // without this the test would pass on a single attempt that never retried + assertTrue(attempts.get() > 1, + "the time must have been spent on retries, but only " + attempts.get() + " attempt was made"); }); } } @@ -253,11 +247,8 @@ void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServ withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "200", () -> { HttpExecution execution = execution(mockServer).expect(200); - long start = System.currentTimeMillis(); assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); - long elapsedMs = System.currentTimeMillis() - start; - assertTrue(elapsedMs < 800, - "expected retry loop to abort near the 200ms max total duration, took " + elapsedMs + "ms"); + mockServer.verify(request().withPath(PATH), VerificationTimes.atMost(5)); }); } @@ -286,6 +277,15 @@ private interface SocketHandler { } /** Serves the socket on a daemon thread until it is closed, then releases what it accepted. */ + /** Waits until the thread reaches the given state, so an interrupt lands where the test means it to. */ + private static void awaitState(Thread thread, Thread.State state) { + long deadline = System.currentTimeMillis() + 10_000; + while (thread.getState() != state && System.currentTimeMillis() < deadline) { + Thread.onSpinWait(); + } + assertEquals(state, thread.getState(), "the worker never reached " + state); + } + private static void startAcceptor(ServerSocket server, SocketHandler handler) { Thread acceptor = new Thread(() -> { List accepted = new ArrayList<>(); diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java index f831130e17..d365f4a826 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java @@ -596,6 +596,22 @@ void testTopicDeleteSuccess(ClientAndServer mockServer) throws Exception { }); } + /** An empty 200 used to reach the response fields and throw NPE instead of answering "nothing deleted". */ + @Test + void testTopicDeleteEmptyBody(ClientAndServer mockServer) throws Exception { + withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { + withProp(Env.PROP_MAAS_AGENT_URL, "http://localhost:" + mockServer.getPort(), () -> { + + mockServer.when( + request().withMethod("DELETE").withPath("/api/v2/kafka/topic"), Times.once() + ).respond(response().withBody("")); + + KafkaMaaSClient kafkaClient = new MaaSAPIClientImpl(() -> "faketoken", null, null).getKafkaClient(); + assertFalse(kafkaClient.deleteTopic(new Classifier("orders"))); + }); + }); + } + @Test void testTopicDeleteError(ClientAndServer mockServer) throws Exception { withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index 3d9019e7e0..70f21f6af3 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -2,6 +2,7 @@ import static com.netcracker.cloud.maas.client.Utils.withProp; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -71,17 +72,27 @@ void stopClientAndStub() { @Test void watchWindowStaysBelowTheReadTimeout() { // the invariant, checked across the range rather than at one point - for (long readTimeoutSeconds : new long[]{2, 5, 6, 10, 30, 60, 120}) { - Duration window = KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(readTimeoutSeconds)); - assertTrue(window.getSeconds() < readTimeoutSeconds, + for (long readTimeoutSeconds : new long[]{1, 2, 5, 6, 10, 30, 60, 120}) { + Duration readTimeout = Duration.ofSeconds(readTimeoutSeconds); + Duration window = KafkaMaaSClientImpl.watchTimeout(readTimeout); + assertTrue(window.compareTo(readTimeout) < 0, "a " + readTimeoutSeconds + "s read timeout must leave room for the answer, got " + window); - assertTrue(window.getSeconds() >= 1, - "the window travels in whole seconds, so it must not round down to zero: " + window); + assertFalse(window.isZero() || window.isNegative(), + "the window must stay positive, got " + window); } assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)), "the default read timeout should keep the full margin"); } + @Test + void backoffGrowsWithConsecutiveFailures() { + assertEquals(1_000, KafkaMaaSClientImpl.watchBackoffMillis(1)); + assertEquals(2_000, KafkaMaaSClientImpl.watchBackoffMillis(2)); + assertEquals(3_000, KafkaMaaSClientImpl.watchBackoffMillis(3)); + assertEquals(30_000, KafkaMaaSClientImpl.watchBackoffMillis(30), "capped"); + assertEquals(30_000, KafkaMaaSClientImpl.watchBackoffMillis(1_000), "stays at the cap"); + } + @Test void failingWatchPollIsBackedOffInsteadOfHotLooping() { withProp(Env.PROP_NAMESPACE, NAMESPACE, () -> { @@ -94,23 +105,19 @@ void failingWatchPollIsBackedOffInsteadOfHotLooping() { "the watch loop reached the agent stub only " + pollMillis.size() + " times out of " + OBSERVED_POLLS + ", so nothing was measured"); - long firstPause = pollMillis.get(1) - pollMillis.get(0); - long secondPause = pollMillis.get(2) - pollMillis.get(1); - // A hot loop would show pauses near zero; a fixed delay would show two equal ones. - assertTrue(firstPause > 500, - "expected the watch loop to pause after a failure, but it polled again in " + firstPause + "ms"); - assertTrue(secondPause > firstPause, - "expected the pause to grow with consecutive failures, but got " - + firstPause + "ms then " + secondPause + "ms"); + for (int poll = 1; poll < OBSERVED_POLLS; poll++) { + long pause = pollMillis.get(poll) - pollMillis.get(poll - 1); + assertTrue(pause > 500, + "expected the watch loop to pause after a failure, but poll " + poll + + " followed the previous one in " + pause + "ms"); + } }); }); } private static KafkaMaaSClientImpl createKafkaClient(String agentUrl) { - System.setProperty(Env.PROP_MAAS_AGENT_URL, agentUrl); var httpClient = HttpClient.getMaasClient(() -> "faketoken"); var serverApiVersion = new ServerApiVersion(httpClient, agentUrl); - System.clearProperty(Env.PROP_MAAS_AGENT_URL); return new KafkaMaaSClientImpl(httpClient, () -> { throw new UnsupportedOperationException("tenant manager is not used in this test"); }, From ec5ed4a66eac18e7b6ec12a0273c5041ab0c9ec4 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 16:41:45 +0400 Subject: [PATCH 20/24] chore: revert fix with semaphore --- .../impl/kafka/KafkaMaaSClientImpl.java | 36 ++++++++++++------- .../impl/http/HttpExecutionFailoverTest.java | 2 +- .../impl/kafka/KafkaMaaSClientImplTest.java | 2 +- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index ca97454cf6..f976bec617 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -13,8 +13,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -71,7 +69,11 @@ static long watchBackoffMillis(int failures) { private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); private volatile boolean closed = false; - private final Semaphore watchSignal = new Semaphore(0); + /** + * Monitor for parking the watch thread. Not the thread itself: {@link Thread#join()} waits on + * that monitor too, and would steal the notification meant for the loop. + */ + private final Object watchLock = new Object(); private final Lazy watchThread = new Lazy<>(() -> { Thread exec = new Thread(this::watchTenantCreateTopics, "watchTopicCreate"); exec.setDaemon(true); @@ -232,7 +234,11 @@ private void deliver(List found) { private boolean parkUntilThereIsSomethingToWatch() { try { log.info("Nothing to watch, sleep thread."); - watchSignal.acquire(); + synchronized (watchLock) { + while (!closed && topicCreateListeners.isEmpty()) { + watchLock.wait(); + } + } log.info("Woke up!"); return true; } catch (InterruptedException e) { @@ -242,19 +248,19 @@ private boolean parkUntilThereIsSomethingToWatch() { } /** - * Linear, capped backoff between failed watch polls, reset on every success. A permit released - * by close() ends the pause early; one released by a new watch does not shorten it, because - * the permit is put back. + * Linear, capped backoff between failed watch polls, reset on every success. Waits on the + * watch monitor rather than sleeping, so close() cuts the pause short. * * @return false if the client was closed or the thread interrupted, meaning the caller stops */ private boolean awaitWatchBackoff(int failures) { long deadline = System.currentTimeMillis() + watchBackoffMillis(failures); try { - for (long left = watchBackoffMillis(failures); !closed && left > 0; - left = deadline - System.currentTimeMillis()) { - if (watchSignal.tryAcquire(left, TimeUnit.MILLISECONDS) && !closed) { - watchSignal.release(); // a new watch, not a close: keep the permit for the park + synchronized (watchLock) { + // waits out the whole pause: only close() ends it early, a new watch does not + for (long left = watchBackoffMillis(failures); !closed && left > 0; + left = deadline - System.currentTimeMillis()) { + watchLock.wait(left); } } return !closed; @@ -276,7 +282,9 @@ public void watchTopicCreate(String name, Consumer callback) { log.info("Add watch for topic by: {}, callback: {}", name, callback); topicCreateListeners.computeIfAbsent(new Classifier(name), k -> Collections.synchronizedList(new ArrayList<>())).add(callback); watchThread.get(); // start the thread if this is the first watch - watchSignal.release(); + synchronized (watchLock) { + watchLock.notifyAll(); + } } @Override @@ -337,7 +345,9 @@ public List search(SearchCriteria criteria) { @Override public void close() { closed = true; - watchSignal.release(); // release the watch thread if it is parked or backing off + synchronized (watchLock) { + watchLock.notifyAll(); // release the watch thread if it is parked or backing off + } if (watchThread.isInitialized()) { watchThread.get().interrupt(); try { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 768e885e36..f272384516 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -276,7 +276,6 @@ private interface SocketHandler { void handle(Socket socket) throws IOException; } - /** Serves the socket on a daemon thread until it is closed, then releases what it accepted. */ /** Waits until the thread reaches the given state, so an interrupt lands where the test means it to. */ private static void awaitState(Thread thread, Thread.State state) { long deadline = System.currentTimeMillis() + 10_000; @@ -286,6 +285,7 @@ private static void awaitState(Thread thread, Thread.State state) { assertEquals(state, thread.getState(), "the worker never reached " + state); } + /** Serves the socket on a daemon thread until it is closed, then releases what it accepted. */ private static void startAcceptor(ServerSocket server, SocketHandler handler) { Thread acceptor = new Thread(() -> { List accepted = new ArrayList<>(); diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java index d365f4a826..232faf721e 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java @@ -598,7 +598,7 @@ void testTopicDeleteSuccess(ClientAndServer mockServer) throws Exception { /** An empty 200 used to reach the response fields and throw NPE instead of answering "nothing deleted". */ @Test - void testTopicDeleteEmptyBody(ClientAndServer mockServer) throws Exception { + void testTopicDeleteEmptyBody(ClientAndServer mockServer) { withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { withProp(Env.PROP_MAAS_AGENT_URL, "http://localhost:" + mockServer.getPort(), () -> { From 4774f00105b5f39d823c8d21f92f68c807c29fc4 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Thu, 3 Sep 2026 17:21:29 +0400 Subject: [PATCH 21/24] chore: removed not required change --- .../maas/client/impl/ApiUrlProvider.java | 2 +- .../maas/client/impl/http/HttpExecution.java | 9 ++------ .../impl/kafka/KafkaMaaSClientImpl.java | 22 ++++++++++--------- .../impl/http/HttpExecutionFailoverTest.java | 2 +- .../KafkaMaaSClientWatchBackoffTest.java | 3 +-- 5 files changed, 17 insertions(+), 21 deletions(-) diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java index 6299fff692..c14809c8a6 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/ApiUrlProvider.java @@ -37,7 +37,7 @@ public String getKafkaTopicSearchUrl() { } public String getKafkaTopicWatchCreateUrl(Duration timeout) { - return String.format("%s/api/v2/kafka/topic/watch-create?timeout=%dms", maasAgentUrl, timeout.toMillis()); + return String.format("%s/api/v2/kafka/topic/watch-create?timeout=%ds", maasAgentUrl, timeout.getSeconds()); } public String getKafkaTopicGetByClassifierUrl() { diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index 78f93e8435..ccaf6dc91a 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -132,9 +132,6 @@ private static boolean isDatabaseUnavailable(String body) { || reason.contains("not in 'active' mode"); } - /** Keeps the client's own timeouts for an attempt that is not bounded by a total duration. */ - private static final long NO_CLAMP = -1; - /** First backoff pause, and the fraction of the total duration a single pause may reach. */ private static final Duration BASE_DELAY = Duration.ofSeconds(1); private static final int MAX_DELAY_FRACTION_OF_TOTAL = 4; @@ -202,9 +199,7 @@ private RetryPolicy> retryPolicy(Request compiledReq, long maxT /** One request/response exchange. Throws {@link RetryableStatus} for an outcome worth repeating. */ private Optional attempt(Request compiledReq, long remainingMs, boolean retrying) throws IOException { Call call = httpClient.newCall(compiledReq); - if (remainingMs >= 0) { - // an attempt starting near the deadline must not overrun the total duration; Failsafe - // truncates the last backoff to land on it, so the last attempt gets the 1ms floor + if (retrying) { call.timeout().timeout(Math.max(1, remainingMs), TimeUnit.MILLISECONDS); } try (Response response = call.execute()) { @@ -234,7 +229,7 @@ private Optional attempt(Request compiledReq, long remainingMs, boolean /** The {@link #noRetry()} path, and a total duration configured to zero. */ private Optional attemptOnce(Request compiledReq) { try { - return attempt(compiledReq, NO_CLAMP, false); + return attempt(compiledReq, 0, false); } catch (IOException e) { throw new MaaSHttpException("Error executing " + compiledReq, e); } diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index f976bec617..eb37737e4c 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -50,12 +50,12 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { private final Duration watchTimeout = watchTimeout(Env.httpTimeout()); /** Largest gap left between the watch window and the read timeout. */ - private static final Duration MAX_WATCH_MARGIN = Duration.ofSeconds(5); + private static final long MAX_WATCH_MARGIN_SECONDS = 5; - /** Half the read timeout at most, so even a one-second timeout keeps the window under it. */ static Duration watchTimeout(Duration httpTimeout) { - Duration margin = httpTimeout.dividedBy(2); - return httpTimeout.minus(margin.compareTo(MAX_WATCH_MARGIN) < 0 ? margin : MAX_WATCH_MARGIN); + long timeoutSeconds = httpTimeout.getSeconds(); + long marginSeconds = Math.min(MAX_WATCH_MARGIN_SECONDS, timeoutSeconds / 2); + return Duration.ofSeconds(Math.max(1, timeoutSeconds - marginSeconds)); } private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); @@ -175,7 +175,7 @@ private boolean pollWhileThereIsSomethingToWatch() { if (Thread.currentThread().isInterrupted()) { log.error("Watch thread interrupted without close(). Topic create callbacks for {} " + "will no longer fire; recreate the client to resume watching", - topicCreateListeners.keySet(), e); + watchedClassifiers(), e); return false; } failures++; @@ -193,14 +193,16 @@ private boolean pollWhileThereIsSomethingToWatch() { private static final TypeReference> TOPIC_LIST = new TypeReference<>() { }; - /** One long poll for topics created since the previous call. */ - private List poll(String url) { - Set watched; + private Set watchedClassifiers() { synchronized (topicCreateListeners) { - watched = new HashSet<>(topicCreateListeners.keySet()); + return new HashSet<>(topicCreateListeners.keySet()); } + } + + /** One long poll for topics created since the previous call. */ + private List poll(String url) { return httpClient.request(url) - .post(watched) + .post(watchedClassifiers()) .expect(200) .noRetry() .sendAndReceive(TOPIC_LIST) diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index f272384516..5a4c877c66 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -248,7 +248,7 @@ void testMaxTotalDuration_AbortsBeforeAttemptsExhausted(ClientAndServer mockServ withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "200", () -> { HttpExecution execution = execution(mockServer).expect(200); assertThrows(MaaSHttpException.class, () -> execution.sendAndReceive(String.class)); - mockServer.verify(request().withPath(PATH), VerificationTimes.atMost(5)); + mockServer.verify(request().withPath(PATH), VerificationTimes.atMost(10)); }); } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index 70f21f6af3..2c95d1f7d8 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -71,8 +71,7 @@ void stopClientAndStub() { */ @Test void watchWindowStaysBelowTheReadTimeout() { - // the invariant, checked across the range rather than at one point - for (long readTimeoutSeconds : new long[]{1, 2, 5, 6, 10, 30, 60, 120}) { + for (long readTimeoutSeconds : new long[]{2, 5, 6, 10, 30, 60, 120}) { Duration readTimeout = Duration.ofSeconds(readTimeoutSeconds); Duration window = KafkaMaaSClientImpl.watchTimeout(readTimeout); assertTrue(window.compareTo(readTimeout) < 0, From e75b9380f57d1a372ba1fc5fe2d92674fa0781cd Mon Sep 17 00:00:00 2001 From: Ksiona Date: Fri, 4 Sep 2026 08:25:34 +0400 Subject: [PATCH 22/24] fix: use failsafe retry policy for watch as well --- maas-client/CHANGELOG.md | 70 +++++----- .../impl/kafka/KafkaMaaSClientImpl.java | 123 +++++++++++------- .../impl/kafka/KafkaMaaSClientImplTest.java | 101 +++++++++++++- .../KafkaMaaSClientWatchBackoffTest.java | 24 ++-- 4 files changed, 221 insertions(+), 97 deletions(-) diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 60c5783bf4..3ec09a80a7 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -2,39 +2,45 @@ ## Unreleased * `Features` - - HTTP calls to maas-agent are now retried on retryable status codes, not only on `IOException`. - Retryable: 5xx, 429 and **405**, the last one only when the body carries a maas-service error. - See "Retry behaviour and configuration" in README for why a 4xx is included — without it the - client does not survive a Postgres leader switchover. 401 is not retried: the token source - refreshes on its own schedule, so a retry would re-send the same token. - - Backoff is exponential with jitter instead of a fixed 1s delay, implemented with a new runtime - dependency, `dev.failsafe:failsafe`. It has no transitive dependencies, but services with - dependency convergence rules will see it appear. - - `deleteTopic` is deliberately not retried: it is not idempotent, and a lost response after a - completed delete would make the next attempt report the topic as still present. - - New configuration: `maas.http.retry.max-total-duration-ms` (`60s` by default) — a single - setting bounding the whole call. The attempt count and the backoff growth are derived from - it, so there are no separate knobs to keep consistent. Every attempt is bounded by what is - left of it, so with the defaults the worst case a caller can see is ~60s, not 60s plus one - `maas.http.timeout`. - - The Kafka topic `watch-create` long poll no longer goes through the retry policy - (`HttpExecution.noRetry()`); its own loop got a linear capped backoff instead, so a - down maas-agent is no longer polled in a hot loop. + - Calls to maas-agent survive a database leader switchover. Retryable: `IOException`, 5xx, 429, + and 405 when the body carries a maas-service error — maas-service reports a read-only database + that way, so the usual "fail fast on 4xx" rule does not hold here. 401 is not retried, because + the token source refreshes on its own schedule. See "Retry behaviour and configuration" in + README. + - New configuration: `maas.http.retry.max-total-duration-ms`, 60s by default, bounding a whole + call including retries. Attempt count and backoff growth derive from it, and each attempt is + capped by what is left, so the worst case a caller sees is that duration rather than the + duration plus one `maas.http.timeout`. `0` disables retries; an unreadable or negative value + warns and falls back to the default. * `Behaviour changes` - - **A call that fails with a retryable status now takes longer before failing.** Previously an - unexpected 5xx or 405 threw immediately; it is now retried within the configured limits. - - Interrupting a thread during a retry wait now restores the interrupt flag and aborts the loop, - instead of swallowing `InterruptedException`. - - `KafkaMaaSClient.watchTopicCreate` throws `IllegalStateException` after `close()`, instead of - registering a callback that can never fire. - - Failed calls to maas-agent now throw `MaaSHttpException` instead of a bare `RuntimeException`. - It extends `MaaSException`, which is a `RuntimeException`, so existing `catch` blocks keep working. - Note the widening: `catch (MaaSException)` used to mean a MaaS business error and now also - catches transport failures, such as the agent being unreachable for the whole minute. - - `maas.http.retry.max-total-duration-ms=0` disables retries, leaving a single attempt. An - unreadable or negative value logs a warning and falls back to the 60s default. - - The Kafka watch poll window is derived from `maas.http.timeout` (25s with the defaults) instead - of a fixed 60s that outlasted the read timeout, so tuning `maas.http.timeout` now also moves it. + - **A call that fails with a retryable status now takes longer before failing.** It used to throw + on the first unexpected 5xx or 405; it is now retried within the configured duration. + - Failed calls to maas-agent throw `MaaSHttpException` instead of a bare `RuntimeException`. It + extends `MaaSException`, so existing `catch` blocks keep working — note the widening: + `catch (MaaSException)` used to mean a MaaS business error and now also catches transport + failures. + - `deleteTopic` and `getOrCreateTopic` with `OnTopicExists.FAIL` are not retried: neither is + idempotent, and a lost response after the server completed the operation would make the retry + report a failure that did not happen. + - The Kafka `watch-create` long poll is paced. A down maas-agent used to be re-polled as fast as + the socket could refuse the connection; the poll now backs off exponentially up to 30s with + jitter, so instances that lose the same agent do not all return at the same moment. It keeps + retrying for the life of the client. + - The watch poll window derives from `maas.http.timeout`, 25s with the defaults, instead of a + fixed 60s that outlasted the read timeout and made every quiet poll fail locally. Tuning + `maas.http.timeout` now moves it; below 2s the watch is not usable. + - `KafkaMaaSClient.watchTopicCreate` throws `IllegalStateException` after `close()`, and after the + watch thread has stopped on its own, instead of registering a callback that can never fire. + - Interrupting a thread during a retry wait restores the interrupt flag and aborts, instead of + swallowing `InterruptedException`. +* `Fixed` + - `deleteTopic` threw `NullPointerException` and `search` threw `NoSuchElementException` when + maas-agent answered 200 with an empty body. They now report nothing deleted and no topics found. + - A response without a body no longer throws while the client reads it, in `getOrCreateTopic`, + `deleteTopic` and the error paths. +* `Dependencies` + - New runtime dependency `dev.failsafe:failsafe`, which carries the retry policies. It has no + transitive dependencies, but services with dependency convergence rules will see it appear. ## 10.0.0 * `Features` diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index eb37737e4c..067be9a662 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -24,6 +24,7 @@ import com.netcracker.cloud.maas.client.api.kafka.SearchCriteria; import com.netcracker.cloud.maas.client.api.kafka.TopicAddress; import com.netcracker.cloud.maas.client.api.kafka.TopicCreateOptions; +import com.netcracker.cloud.maas.client.api.kafka.protocolextractors.OnTopicExists; import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; import com.netcracker.cloud.maas.client.impl.Env; import com.netcracker.cloud.maas.client.impl.Lazy; @@ -33,8 +34,12 @@ import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicRequest; import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicTemplate; import com.netcracker.cloud.maas.client.impl.http.HttpClient; +import com.netcracker.cloud.maas.client.impl.http.HttpExecution; import com.netcracker.cloud.tenantmanager.client.TenantManagerConnector; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -52,6 +57,10 @@ public class KafkaMaaSClientImpl implements KafkaMaaSClient { /** Largest gap left between the watch window and the read timeout. */ private static final long MAX_WATCH_MARGIN_SECONDS = 5; + /** + * The margin is clamped, so a small read timeout narrows the window instead of inverting it. + * The window travels in whole seconds, so {@code maas.http.timeout} below 2s is unsupported. + */ static Duration watchTimeout(Duration httpTimeout) { long timeoutSeconds = httpTimeout.getSeconds(); long marginSeconds = Math.min(MAX_WATCH_MARGIN_SECONDS, timeoutSeconds / 2); @@ -60,15 +69,20 @@ static Duration watchTimeout(Duration httpTimeout) { private static final Duration WATCH_RETRY_INTERVAL = Duration.ofSeconds(1); private static final Duration WATCH_MAX_RETRY_INTERVAL = Duration.ofSeconds(30); + private static final double WATCH_BACKOFF_MULTIPLIER = 2.0; - /** Grows by one interval per consecutive failure, up to the cap. */ - static long watchBackoffMillis(int failures) { - return Math.min(failures * WATCH_RETRY_INTERVAL.toMillis(), WATCH_MAX_RETRY_INTERVAL.toMillis()); - } + /** Keeps instances that lost the same agent from returning to it at the same moment. */ + private static final double WATCH_BACKOFF_JITTER = 0.2; // there is no need in highly concurrent map/lists implementation, we will wait for network responses most of the time private final Map>> topicCreateListeners = Collections.synchronizedMap(new HashMap<>()); private volatile boolean closed = false; + /** + * Set when the watch thread exits without {@link #close()}. Nothing restarts it, so a later + * {@link #watchTopicCreate} must refuse rather than register a callback that cannot fire. + */ + private volatile boolean watchThreadDead = false; + /** * Monitor for parking the watch thread. Not the thread itself: {@link Thread#join()} waits on * that monitor too, and would steal the notification meant for the loop. @@ -99,9 +113,13 @@ public TopicAddress getOrCreateTopic(Classifier classifier, TopicCreateOptions o String url = apiProvider.getKafkaTopicUrl(options.getOnTopicExists()); log.info("Get or create topic by classifier=`{}' and options=`{}'", classifier, options); - return httpClient.request(url) + HttpExecution request = httpClient.request(url) .post(TopicRequest.builder(classifier).build().options(options)) - .expect(HTTP_OK, HTTP_CREATED) + .expect(HTTP_OK, HTTP_CREATED); + if (options.getOnTopicExists() == OnTopicExists.FAIL) { + request.noRetry(); + } + return request .sendAndReceive(TopicInfo.class) .map(TopicAddressImpl::new) .get(); @@ -144,12 +162,19 @@ public void watchTenantTopics(String name, Consumer> callback } private void watchTenantCreateTopics() { - while (!closed) { - if (!pollWhileThereIsSomethingToWatch()) { - return; + try { + while (!closed) { + if (!pollWhileThereIsSomethingToWatch()) { + return; + } + if (closed || !parkUntilThereIsSomethingToWatch()) { + return; + } } - if (closed || !parkUntilThereIsSomethingToWatch()) { - return; + } finally { + if (!closed) { + // exiting without close(): let watchTopicCreate refuse further registrations + watchThreadDead = true; } } } @@ -160,14 +185,17 @@ private void watchTenantCreateTopics() { * @return false if the thread must stop */ private boolean pollWhileThereIsSomethingToWatch() { - int failures = 0; while (!closed && !topicCreateListeners.isEmpty()) { String url = apiProvider.getKafkaTopicWatchCreateUrl(watchTimeout); List found; try { - found = poll(url); - failures = 0; + found = Failsafe.with(watchRetryPolicy(url)).get(() -> poll(url)); } catch (Exception e) { + Throwable cause = e instanceof FailsafeException failsafe ? failsafe.getCause() : e; + if (cause instanceof InterruptedException) { + // Failsafe clears the flag when it catches this, so the check below cannot see it + Thread.currentThread().interrupt(); + } // `closed` is checked too: an interrupt can be swallowed further down if (closed) { return false; // shutting down, not a failure worth reporting @@ -175,21 +203,36 @@ private boolean pollWhileThereIsSomethingToWatch() { if (Thread.currentThread().isInterrupted()) { log.error("Watch thread interrupted without close(). Topic create callbacks for {} " + "will no longer fire; recreate the client to resume watching", - watchedClassifiers(), e); + watchedNames(), cause); return false; } - failures++; - log.warn("Error execute request to {}. Attempt {}, will back off before retrying", url, failures, e); - if (!awaitWatchBackoff(failures)) { - return false; // closed or interrupted while backing off - } - continue; // nothing was received, nothing to deliver + // the policy only gives up on interrupt, so this should be unreachable + log.error("Watch poll for {} stopped unexpectedly. Topic create callbacks for {} " + + "will no longer fire; recreate the client to resume watching", + url, watchedNames(), cause); + return false; } deliver(found); } return true; } + /** + * Backoff between failed watch polls. Unbounded in attempts, because the subscription lives as + * long as the client; a fresh policy per poll resets the sequence after every success, and the + * interrupt from {@link #close()} ends the wait. + */ + private RetryPolicy> watchRetryPolicy(String url) { + return RetryPolicy.>builder() + .handle(Exception.class) + .withBackoff(WATCH_RETRY_INTERVAL, WATCH_MAX_RETRY_INTERVAL, WATCH_BACKOFF_MULTIPLIER) + .withJitter(WATCH_BACKOFF_JITTER) + .withMaxAttempts(-1) + .onRetry(event -> log.warn("Error execute request to {}. Attempt {} failed with {}, will back off before retrying", + url, event.getAttemptCount(), event.getLastException())) + .build(); + } + private static final TypeReference> TOPIC_LIST = new TypeReference<>() { }; @@ -199,6 +242,11 @@ private Set watchedClassifiers() { } } + /** Names only: a classifier is an open map and may carry whatever the caller put in it. */ + private List watchedNames() { + return watchedClassifiers().stream().map(Classifier::getName).toList(); + } + /** One long poll for topics created since the previous call. */ private List poll(String url) { return httpClient.request(url) @@ -249,36 +297,15 @@ private boolean parkUntilThereIsSomethingToWatch() { } } - /** - * Linear, capped backoff between failed watch polls, reset on every success. Waits on the - * watch monitor rather than sleeping, so close() cuts the pause short. - * - * @return false if the client was closed or the thread interrupted, meaning the caller stops - */ - private boolean awaitWatchBackoff(int failures) { - long deadline = System.currentTimeMillis() + watchBackoffMillis(failures); - try { - synchronized (watchLock) { - // waits out the whole pause: only close() ends it early, a new watch does not - for (long left = watchBackoffMillis(failures); !closed && left > 0; - left = deadline - System.currentTimeMillis()) { - watchLock.wait(left); - } - } - return !closed; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - @Override public void watchTopicCreate(String name, Consumer callback) { if (closed) { - // the watch thread has already exited and nothing restarts it, so the callback - // would never fire throw new IllegalStateException("Client is closed, cannot watch topic: " + name); } + if (watchThreadDead) { + throw new IllegalStateException("Watch thread has stopped unexpectedly, cannot watch topic: " + + name + "; recreate the client to resume watching"); + } apiProvider.getServerApiVersion().requiresApiVersion(2, 8); log.info("Add watch for topic by: {}, callback: {}", name, callback); @@ -348,10 +375,10 @@ public List search(SearchCriteria criteria) { public void close() { closed = true; synchronized (watchLock) { - watchLock.notifyAll(); // release the watch thread if it is parked or backing off + watchLock.notifyAll(); // release the watch thread if it is parked } if (watchThread.isInitialized()) { - watchThread.get().interrupt(); + watchThread.get().interrupt(); // also cuts a backoff sleep short try { watchThread.get().join(1000); } catch (InterruptedException e) { diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java index 232faf721e..dbe6bcb580 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java @@ -21,6 +21,7 @@ import org.mockserver.matchers.Times; import org.mockserver.mock.action.ExpectationResponseCallback; import org.mockserver.model.HttpRequest; +import org.mockserver.verify.VerificationTimes; import com.netcracker.cloud.maas.client.Utils; import com.netcracker.cloud.maas.client.api.Classifier; @@ -450,10 +451,12 @@ public void testGetOrCreateTopicWithRetry(ClientAndServer mockServer) throws IOE "}\n") ); - // run test + // run test. MERGE is what keeps this recoverable: hitting an already-created + // topic again is harmless, so the retry after the timed-out first attempt is safe. var client = createKafkaClient("http://localhost:" + mockServer.getPort()); TopicAddress topicAddress = client.getOrCreateTopic(new Classifier("orders"), TopicCreateOptions.builder() + .onTopicExists(OnTopicExists.MERGE) .name("user-test1") .build()); assertEquals("user-test1", topicAddress.getTopicName()); @@ -461,6 +464,49 @@ public void testGetOrCreateTopicWithRetry(ClientAndServer mockServer) throws IOE }); } + /** + * Under FAIL semantics a retry after a locally-timed-out request could be replaying a create + * that already succeeded server-side, and would fail permanently against an existing topic. + * So, unlike {@link #testGetOrCreateTopicWithRetry}, this must not retry and must surface the + * timeout instead of silently recovering. + */ + @Test + public void testGetOrCreateTopicFailOnExistsDoesNotRetry(ClientAndServer mockServer) throws IOException { + withProp(Env.PROP_NAMESPACE, "cloudbss-kube-core-demo-2", () -> { + withProp(Env.PROP_HTTP_TIMEOUT, "1", () -> { + + mockServer.when( + request() + .withPath("/api/v2/kafka/topic"), + Times.once() + ).respond( + response() + .withStatusCode(200) + .withDelay(TimeUnit.SECONDS, 2) + ); + + mockServer.when( + request() + .withPath("/api/v2/kafka/topic"), + Times.once() + ).respond( + response() + .withStatusCode(200) + .withBody("{\n" + + " \"name\": \"user-test1\"\n" + + "}\n") + ); + + var client = createKafkaClient("http://localhost:" + mockServer.getPort()); + assertThrows(MaaSException.class, () -> client.getOrCreateTopic(new Classifier("orders"), + TopicCreateOptions.builder() + .onTopicExists(OnTopicExists.FAIL) + .name("user-test1") + .build())); + }); + }); + } + @Test public void testGetOrCreateTopicV2(ClientAndServer mockServer) { // prepare environment @@ -612,6 +658,49 @@ void testTopicDeleteEmptyBody(ClientAndServer mockServer) { }); } + /** + * A delete is not idempotent: a retry after the server completed one comes back with empty + * lists and would report the topic as still present. + */ + @Test + void testTopicDeleteIsNotRetried(ClientAndServer mockServer) { + withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { + withProp(Env.PROP_MAAS_AGENT_URL, "http://localhost:" + mockServer.getPort(), () -> { + + mockServer.when(request().withMethod("DELETE").withPath("/api/v2/kafka/topic"), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + KafkaMaaSClient kafkaClient = new MaaSAPIClientImpl(() -> "faketoken", null, null).getKafkaClient(); + assertThrows(MaaSException.class, () -> kafkaClient.deleteTopic(new Classifier("orders"))); + + mockServer.verify(request().withMethod("DELETE").withPath("/api/v2/kafka/topic"), + VerificationTimes.exactly(1)); + }); + }); + } + + /** Same reasoning as the delete: under FAIL a retry hits the topic the first attempt created. */ + @Test + void testGetOrCreateTopicIsNotRetriedWhenItMustFailOnExisting(ClientAndServer mockServer) { + withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { + withProp(Env.PROP_MAAS_AGENT_URL, "http://localhost:" + mockServer.getPort(), () -> { + + mockServer.when(request().withMethod("POST").withPath("/api/v2/kafka/topic"), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + + KafkaMaaSClient kafkaClient = new MaaSAPIClientImpl(() -> "faketoken", null, null).getKafkaClient(); + TopicCreateOptions failOnExisting = TopicCreateOptions.builder() + .onTopicExists(OnTopicExists.FAIL) + .build(); + assertThrows(MaaSException.class, + () -> kafkaClient.getOrCreateTopic(new Classifier("orders"), failOnExisting)); + + mockServer.verify(request().withMethod("POST").withPath("/api/v2/kafka/topic"), + VerificationTimes.exactly(1)); + }); + }); + } + @Test void testTopicDeleteError(ClientAndServer mockServer) throws Exception { withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { @@ -791,6 +880,16 @@ void testClose(ClientAndServer mockServer) { }); } + @Test + void testWatchTopicCreateThrowsAfterClose(ClientAndServer mockServer) { + withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { + KafkaMaaSClientImpl client = createKafkaClient("http://localhost:" + mockServer.getPort()); + client.close(); + + assertThrows(IllegalStateException.class, () -> client.watchTopicCreate("orders", addr -> {})); + }); + } + private KafkaMaaSClientImpl createKafkaClient(String agentUrl) { System.setProperty(Env.PROP_MAAS_AGENT_URL, agentUrl); var httpClient = HttpClient.getMaasClient(() -> "faketoken"); diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java index 2c95d1f7d8..9f0d1572fb 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientWatchBackoffTest.java @@ -43,6 +43,7 @@ class KafkaMaaSClientWatchBackoffTest { private static final int OBSERVED_POLLS = 3; private final List pollMillis = Collections.synchronizedList(new ArrayList<>()); + private final List pollQueries = Collections.synchronizedList(new ArrayList<>()); private final CountDownLatch pollsObserved = new CountDownLatch(OBSERVED_POLLS); private HttpServer agentStub; private KafkaMaaSClientImpl client; @@ -64,13 +65,11 @@ void stopClientAndStub() { } /** - * maas-service holds a watch poll open for the whole requested window and then answers 200 - * with an empty list. If the window outlasts the client read timeout, that answer never - * arrives: every quiet poll fails locally, walks the backoff up to its 30s cap and delays - * the next real topic-create event. + * A window that outlasts the read timeout means the empty 200 ending a quiet poll never + * arrives, so every such poll fails locally and walks the backoff up to its cap. */ @Test - void watchWindowStaysBelowTheReadTimeout() { + void watchWindowFormulaStaysBelowTheReadTimeout() { for (long readTimeoutSeconds : new long[]{2, 5, 6, 10, 30, 60, 120}) { Duration readTimeout = Duration.ofSeconds(readTimeoutSeconds); Duration window = KafkaMaaSClientImpl.watchTimeout(readTimeout); @@ -79,17 +78,6 @@ void watchWindowStaysBelowTheReadTimeout() { assertFalse(window.isZero() || window.isNegative(), "the window must stay positive, got " + window); } - assertEquals(Duration.ofSeconds(25), KafkaMaaSClientImpl.watchTimeout(Duration.ofSeconds(30)), - "the default read timeout should keep the full margin"); - } - - @Test - void backoffGrowsWithConsecutiveFailures() { - assertEquals(1_000, KafkaMaaSClientImpl.watchBackoffMillis(1)); - assertEquals(2_000, KafkaMaaSClientImpl.watchBackoffMillis(2)); - assertEquals(3_000, KafkaMaaSClientImpl.watchBackoffMillis(3)); - assertEquals(30_000, KafkaMaaSClientImpl.watchBackoffMillis(30), "capped"); - assertEquals(30_000, KafkaMaaSClientImpl.watchBackoffMillis(1_000), "stays at the cap"); } @Test @@ -110,6 +98,9 @@ void failingWatchPollIsBackedOffInsteadOfHotLooping() { "expected the watch loop to pause after a failure, but poll " + poll + " followed the previous one in " + pause + "ms"); } + + assertEquals("timeout=25s", pollQueries.get(0), + "the poll must carry the window derived from maas.http.timeout"); }); }); } @@ -126,6 +117,7 @@ private static KafkaMaaSClientImpl createKafkaClient(String agentUrl) { /** Answers every poll with 500, the code maas-agent returns when it cannot reach maas-service. */ private void failWatchPoll(HttpExchange exchange) throws IOException { pollMillis.add(System.currentTimeMillis()); + pollQueries.add(exchange.getRequestURI().getQuery()); pollsObserved.countDown(); respond(exchange, 500, "{\"error\":\"error proxying request: maas-service unavailable\"}"); } From 37343ed75e0b05dd48a7e30b058c3cab89beb3dc Mon Sep 17 00:00:00 2001 From: Ksiona Date: Fri, 4 Sep 2026 09:09:56 +0400 Subject: [PATCH 23/24] fix: revert to retry for getOrCreateTopic --- maas-client/CHANGELOG.md | 7 +- maas-client/README.md | 6 ++ .../impl/kafka/KafkaMaaSClientImpl.java | 58 +++++++------ .../impl/kafka/KafkaMaaSClientImplTest.java | 83 +++++-------------- 4 files changed, 60 insertions(+), 94 deletions(-) diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 3ec09a80a7..5a65356b54 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -19,9 +19,10 @@ extends `MaaSException`, so existing `catch` blocks keep working — note the widening: `catch (MaaSException)` used to mean a MaaS business error and now also catches transport failures. - - `deleteTopic` and `getOrCreateTopic` with `OnTopicExists.FAIL` are not retried: neither is - idempotent, and a lost response after the server completed the operation would make the retry - report a failure that did not happen. + - **`deleteTopic` is not retried, on any options.** It used to be, as any other call. Its response + carries how many topics were deleted, and a repeat of a delete whose response was lost reports + zero for a topic that is already gone. Callers that need the old behaviour must retry themselves + and treat `false` as "not found" rather than "not deleted". - The Kafka `watch-create` long poll is paced. A down maas-agent used to be re-polled as fast as the socket could refuse the connection; the poll now backs off exponentially up to 30s with jitter, so instances that lose the same agent do not all return at the same moment. It keeps diff --git a/maas-client/README.md b/maas-client/README.md index c48faa5d00..f76804671a 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -88,6 +88,12 @@ below it — maas-service holds the request open for the whole window and then a with an empty list, which the client has to be able to receive. With the default 30s timeout the window is 25s. +`deleteTopic` is excluded as well, on any options: its response carries how many +topics were deleted, and a repeat of a delete whose response was lost reports zero +for a topic that is already gone. `getOrCreateTopic` is retried on any options — +maas-service resolves the classifier before it looks at `onTopicExists`, so a +repeated create returns the registration the first attempt made. + Which responses are retried: | Response | Retried | Why | diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 067be9a662..30dfef288d 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -24,7 +24,6 @@ import com.netcracker.cloud.maas.client.api.kafka.SearchCriteria; import com.netcracker.cloud.maas.client.api.kafka.TopicAddress; import com.netcracker.cloud.maas.client.api.kafka.TopicCreateOptions; -import com.netcracker.cloud.maas.client.api.kafka.protocolextractors.OnTopicExists; import com.netcracker.cloud.maas.client.impl.ApiUrlProvider; import com.netcracker.cloud.maas.client.impl.Env; import com.netcracker.cloud.maas.client.impl.Lazy; @@ -34,7 +33,6 @@ import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicRequest; import com.netcracker.cloud.maas.client.impl.dto.kafka.v1.TopicTemplate; import com.netcracker.cloud.maas.client.impl.http.HttpClient; -import com.netcracker.cloud.maas.client.impl.http.HttpExecution; import com.netcracker.cloud.tenantmanager.client.TenantManagerConnector; import dev.failsafe.Failsafe; @@ -113,13 +111,12 @@ public TopicAddress getOrCreateTopic(Classifier classifier, TopicCreateOptions o String url = apiProvider.getKafkaTopicUrl(options.getOnTopicExists()); log.info("Get or create topic by classifier=`{}' and options=`{}'", classifier, options); - HttpExecution request = httpClient.request(url) + // Retried on the default options too: maas-service resolves the classifier first, under a + // lock, and only consults onTopicExists for a topic missing from its registry. A repeat of + // a create whose response was lost therefore returns the registration the first one made. + return httpClient.request(url) .post(TopicRequest.builder(classifier).build().options(options)) - .expect(HTTP_OK, HTTP_CREATED); - if (options.getOnTopicExists() == OnTopicExists.FAIL) { - request.noRetry(); - } - return request + .expect(HTTP_OK, HTTP_CREATED) .sendAndReceive(TopicInfo.class) .map(TopicAddressImpl::new) .get(); @@ -130,6 +127,10 @@ public Optional getTopic(Classifier classifier) { return Optional.ofNullable(searchTopic(classifier)); } + /** + * Not retried: the response says how many topics were deleted, and a repeat of a delete whose + * response was lost reports zero for a topic that is gone. + */ @Override public boolean deleteTopic(Classifier classifier) { TopicDeleteResponse resp = httpClient.request(apiProvider.getKafkaTopicUrl(null)) @@ -191,25 +192,7 @@ private boolean pollWhileThereIsSomethingToWatch() { try { found = Failsafe.with(watchRetryPolicy(url)).get(() -> poll(url)); } catch (Exception e) { - Throwable cause = e instanceof FailsafeException failsafe ? failsafe.getCause() : e; - if (cause instanceof InterruptedException) { - // Failsafe clears the flag when it catches this, so the check below cannot see it - Thread.currentThread().interrupt(); - } - // `closed` is checked too: an interrupt can be swallowed further down - if (closed) { - return false; // shutting down, not a failure worth reporting - } - if (Thread.currentThread().isInterrupted()) { - log.error("Watch thread interrupted without close(). Topic create callbacks for {} " - + "will no longer fire; recreate the client to resume watching", - watchedNames(), cause); - return false; - } - // the policy only gives up on interrupt, so this should be unreachable - log.error("Watch poll for {} stopped unexpectedly. Topic create callbacks for {} " - + "will no longer fire; recreate the client to resume watching", - url, watchedNames(), cause); + reportWatchStopped(url, e); return false; } deliver(found); @@ -217,6 +200,27 @@ private boolean pollWhileThereIsSomethingToWatch() { return true; } + /** Reports why the watch thread is stopping, unless it is a normal {@link #close()}. */ + private void reportWatchStopped(String url, Exception e) { + Throwable cause = e instanceof FailsafeException failsafe ? failsafe.getCause() : e; + if (cause instanceof InterruptedException) { + // Failsafe clears the flag when it catches this, so the check below cannot see it + Thread.currentThread().interrupt(); + } + // `closed` is checked too: an interrupt can be swallowed further down + if (closed) { + return; // shutting down, not a failure worth reporting + } + if (Thread.currentThread().isInterrupted()) { + log.error("Watch thread interrupted without close(). Topic create callbacks for {} " + + "will no longer fire; recreate the client to resume watching", watchedNames(), cause); + return; + } + // the policy only gives up on interrupt, so this should be unreachable + log.error("Watch poll for {} stopped unexpectedly. Topic create callbacks for {} " + + "will no longer fire; recreate the client to resume watching", url, watchedNames(), cause); + } + /** * Backoff between failed watch polls. Unbounded in attempts, because the subscription lives as * long as the client; a fresh policy per poll resets the sequence after every success, and the diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java index dbe6bcb580..db7d3d88bb 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java @@ -451,62 +451,16 @@ public void testGetOrCreateTopicWithRetry(ClientAndServer mockServer) throws IOE "}\n") ); - // run test. MERGE is what keeps this recoverable: hitting an already-created - // topic again is harmless, so the retry after the timed-out first attempt is safe. + // default options: maas-service resolves the classifier before it looks at + // onTopicExists, so the retry after the timed-out first attempt gets the same topic var client = createKafkaClient("http://localhost:" + mockServer.getPort()); TopicAddress topicAddress = client.getOrCreateTopic(new Classifier("orders"), - TopicCreateOptions.builder() - .onTopicExists(OnTopicExists.MERGE) - .name("user-test1") - .build()); + TopicCreateOptions.builder().name("user-test1").build()); assertEquals("user-test1", topicAddress.getTopicName()); }); }); } - /** - * Under FAIL semantics a retry after a locally-timed-out request could be replaying a create - * that already succeeded server-side, and would fail permanently against an existing topic. - * So, unlike {@link #testGetOrCreateTopicWithRetry}, this must not retry and must surface the - * timeout instead of silently recovering. - */ - @Test - public void testGetOrCreateTopicFailOnExistsDoesNotRetry(ClientAndServer mockServer) throws IOException { - withProp(Env.PROP_NAMESPACE, "cloudbss-kube-core-demo-2", () -> { - withProp(Env.PROP_HTTP_TIMEOUT, "1", () -> { - - mockServer.when( - request() - .withPath("/api/v2/kafka/topic"), - Times.once() - ).respond( - response() - .withStatusCode(200) - .withDelay(TimeUnit.SECONDS, 2) - ); - - mockServer.when( - request() - .withPath("/api/v2/kafka/topic"), - Times.once() - ).respond( - response() - .withStatusCode(200) - .withBody("{\n" + - " \"name\": \"user-test1\"\n" + - "}\n") - ); - - var client = createKafkaClient("http://localhost:" + mockServer.getPort()); - assertThrows(MaaSException.class, () -> client.getOrCreateTopic(new Classifier("orders"), - TopicCreateOptions.builder() - .onTopicExists(OnTopicExists.FAIL) - .name("user-test1") - .build())); - }); - }); - } - @Test public void testGetOrCreateTopicV2(ClientAndServer mockServer) { // prepare environment @@ -659,8 +613,8 @@ void testTopicDeleteEmptyBody(ClientAndServer mockServer) { } /** - * A delete is not idempotent: a retry after the server completed one comes back with empty - * lists and would report the topic as still present. + * A retry after the server completed a delete comes back with empty lists, which the client + * reads as "nothing was deleted". */ @Test void testTopicDeleteIsNotRetried(ClientAndServer mockServer) { @@ -671,7 +625,8 @@ void testTopicDeleteIsNotRetried(ClientAndServer mockServer) { .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); KafkaMaaSClient kafkaClient = new MaaSAPIClientImpl(() -> "faketoken", null, null).getKafkaClient(); - assertThrows(MaaSException.class, () -> kafkaClient.deleteTopic(new Classifier("orders"))); + Classifier orders = new Classifier("orders"); + assertThrows(MaaSException.class, () -> kafkaClient.deleteTopic(orders)); mockServer.verify(request().withMethod("DELETE").withPath("/api/v2/kafka/topic"), VerificationTimes.exactly(1)); @@ -679,24 +634,24 @@ void testTopicDeleteIsNotRetried(ClientAndServer mockServer) { }); } - /** Same reasoning as the delete: under FAIL a retry hits the topic the first attempt created. */ + /** Create is the operation a switchover interrupts most often, and it retries on any options. */ @Test - void testGetOrCreateTopicIsNotRetriedWhenItMustFailOnExisting(ClientAndServer mockServer) { + void testGetOrCreateTopicIsRetriedOnDefaultOptions(ClientAndServer mockServer) { withProp(Env.PROP_NAMESPACE, "cloud-dev", () -> { withProp(Env.PROP_MAAS_AGENT_URL, "http://localhost:" + mockServer.getPort(), () -> { + withProp(Env.PROP_HTTP_RETRY_MAX_TOTAL_DURATION_MS, "1000", () -> { - mockServer.when(request().withMethod("POST").withPath("/api/v2/kafka/topic"), Times.unlimited()) - .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); + mockServer.when(request().withMethod("POST").withPath("/api/v2/kafka/topic"), Times.unlimited()) + .respond(response().withStatusCode(500).withBody("{\"error\":\"agent down\"}")); - KafkaMaaSClient kafkaClient = new MaaSAPIClientImpl(() -> "faketoken", null, null).getKafkaClient(); - TopicCreateOptions failOnExisting = TopicCreateOptions.builder() - .onTopicExists(OnTopicExists.FAIL) - .build(); - assertThrows(MaaSException.class, - () -> kafkaClient.getOrCreateTopic(new Classifier("orders"), failOnExisting)); + KafkaMaaSClient kafkaClient = new MaaSAPIClientImpl(() -> "faketoken", null, null).getKafkaClient(); + Classifier orders = new Classifier("orders"); + assertThrows(MaaSException.class, + () -> kafkaClient.getOrCreateTopic(orders, TopicCreateOptions.DEFAULTS)); - mockServer.verify(request().withMethod("POST").withPath("/api/v2/kafka/topic"), - VerificationTimes.exactly(1)); + mockServer.verify(request().withMethod("POST").withPath("/api/v2/kafka/topic"), + VerificationTimes.atLeast(2)); + }); }); }); } From 78f332712043560a037bd2b7ff498b7d30606e05 Mon Sep 17 00:00:00 2001 From: Ksiona Date: Fri, 4 Sep 2026 10:09:09 +0400 Subject: [PATCH 24/24] fix: retryableResponses --- maas-client/CHANGELOG.md | 8 +- maas-client/README.md | 8 +- .../maas/client/impl/http/HttpExecution.java | 25 ++-- .../impl/kafka/KafkaMaaSClientImpl.java | 10 +- .../impl/http/HttpExecutionFailoverTest.java | 15 +- .../impl/kafka/KafkaMaaSClientImplTest.java | 131 +++++------------- 6 files changed, 81 insertions(+), 116 deletions(-) diff --git a/maas-client/CHANGELOG.md b/maas-client/CHANGELOG.md index 5a65356b54..bc18966314 100644 --- a/maas-client/CHANGELOG.md +++ b/maas-client/CHANGELOG.md @@ -3,10 +3,10 @@ ## Unreleased * `Features` - Calls to maas-agent survive a database leader switchover. Retryable: `IOException`, 5xx, 429, - and 405 when the body carries a maas-service error — maas-service reports a read-only database - that way, so the usual "fail fast on 4xx" rule does not hold here. 401 is not retried, because - the token source refreshes on its own schedule. See "Retry behaviour and configuration" in - README. + and 405 when the response reason names a database that cannot be written — maas-service reports + a read-only database that way, so the usual "fail fast on 4xx" rule does not hold here. 401 is + not retried, because the token source refreshes on its own schedule. See "Retry behaviour and + configuration" in README. - New configuration: `maas.http.retry.max-total-duration-ms`, 60s by default, bounding a whole call including retries. Attempt count and backoff growth derive from it, and each attempt is capped by what is left, so the worst case a caller sees is that duration rather than the diff --git a/maas-client/README.md b/maas-client/README.md index f76804671a..c575756c2e 100644 --- a/maas-client/README.md +++ b/maas-client/README.md @@ -101,13 +101,19 @@ Which responses are retried: | `IOException` | yes | connection refused/reset while the agent is being rescheduled | | 5xx | yes | includes the `500` maas-agent returns when it cannot reach maas-service at all | | 429 | yes | throttling | -| **405** | **only with a maas-service error body** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a switchover arrives as `405`, not as `5xx`. A plain `405` — a route removed on the server, an ingress rejecting the method — is permanent and fails fast | +| **405** | **only when the `reason` names a database that cannot be written** | maas-service maps PostgreSQL error `25006` (READ ONLY SQL TRANSACTION) to `405`, so a write against a demoted Patroni node during a switchover arrives as `405`, not as `5xx`. A plain `405` — a route removed on the server, an ingress rejecting the method — is permanent and fails fast | | 401 | no | `CachingTokenSource` refreshes on its own polling interval, so a retry within the backoff reads the same token, and `M2MInterceptor` has already made its own 401 round trip by then | | other 4xx | no | permanent client errors, failed on the first attempt | The 405 entry is deliberate: the usual "retry 5xx, fail fast on 4xx" rule does not survive a database leader switchover here. +The `reason` of the error envelope is what decides, not the error code: every +maas-service error carries the same code, so the envelope alone says nothing. The +match is loose — the reason has to mention a database together with `read-only` +or `not active` — so a reworded message on the server still counts, while a `405` +about a read-only *field* does not. + ## Kafka client usage example All MaaS operations for Kafka is collected in [KafkaMaaSClient](https://github.com/Netcracker/qubership-maas-client/blob/main/client/src/main/java/com/netcracker/cloud/maas/client/api/kafka/KafkaMaaSClient.java). To obtain *new* instance of MaaS Kafka client just call: ```java diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java index ccaf6dc91a..2e7eafa3d7 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/http/HttpExecution.java @@ -107,9 +107,6 @@ public Optional sendAndReceive(OmnivoreFunction responseDeseri return sendAndReceive().map(der(responseDeserializer)); } - /** Code carried by every maas-service TMF error envelope. */ - private static final String MAAS_ERROR_CODE = "MAAS-0600"; - /** * Whether the status is worth another attempt. 405 is here because maas-service reports a * read-only database that way; 401 is not, because the token source refreshes on its own @@ -122,14 +119,26 @@ private static boolean isRetryableStatus(int code, String body) { return code == 405 && isDatabaseUnavailable(body); } - /** Tells the 405 of a read-only database apart from a plain one, which is permanent. */ + /** Wordings of the two maas-service errors a leader switchover produces, and rewordings. */ + private static final List DATABASE_UNAVAILABLE_MARKERS = + List.of("read-only", "read only", "not in 'active' mode", "not active"); + + /** + * Reads the reason of a maas-service error envelope. The word "database" is required next to + * the marker, so an unrelated 405 that happens to mention read-only data stays permanent. + */ private static boolean isDatabaseUnavailable(String body) { - if (body == null || !body.contains(MAAS_ERROR_CODE)) { + if (body == null || body.isEmpty()) { return false; } - String reason = body.toLowerCase(Locale.ROOT); - return reason.contains("read-only") || reason.contains("read only") - || reason.contains("not in 'active' mode"); + String reason; + try { + reason = MAPPER.readTree(body).path("reason").asText("").toLowerCase(Locale.ROOT); + } catch (JsonProcessingException e) { + return false; // not a maas-service envelope + } + return reason.contains("database") + && DATABASE_UNAVAILABLE_MARKERS.stream().anyMatch(reason::contains); } /** First backoff pause, and the fraction of the total duration a single pause may reach. */ diff --git a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java index 30dfef288d..f13e2ed7d5 100644 --- a/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java +++ b/maas-client/client/src/main/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImpl.java @@ -111,9 +111,8 @@ public TopicAddress getOrCreateTopic(Classifier classifier, TopicCreateOptions o String url = apiProvider.getKafkaTopicUrl(options.getOnTopicExists()); log.info("Get or create topic by classifier=`{}' and options=`{}'", classifier, options); - // Retried on the default options too: maas-service resolves the classifier first, under a - // lock, and only consults onTopicExists for a topic missing from its registry. A repeat of - // a create whose response was lost therefore returns the registration the first one made. + // Retried on any options: maas-service resolves the classifier before it looks at + // onTopicExists, so a repeat returns the registration the first attempt made. return httpClient.request(url) .post(TopicRequest.builder(classifier).build().options(options)) .expect(HTTP_OK, HTTP_CREATED) @@ -127,10 +126,7 @@ public Optional getTopic(Classifier classifier) { return Optional.ofNullable(searchTopic(classifier)); } - /** - * Not retried: the response says how many topics were deleted, and a repeat of a delete whose - * response was lost reports zero for a topic that is gone. - */ + /** Not retried: a repeat reports zero deleted for a topic the first attempt already removed. */ @Override public boolean deleteTopic(Classifier classifier) { TopicDeleteResponse resp = httpClient.request(apiProvider.getKafkaTopicUrl(null)) diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java index 5a4c877c66..e4d93fde49 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/http/HttpExecutionFailoverTest.java @@ -46,6 +46,11 @@ static Stream retryableResponses() { return Stream.of( arguments("a read-only database", 405, "{\"code\":\"MAAS-0600\",\"reason\":\"database is in read-only mode\"}", 2), + arguments("a database out of 'active' mode", 405, + "{\"code\":\"MAAS-0600\",\"reason\":\"database is not in 'active' mode\"}", 2), + // the reason is matched loosely, so a reworded one still counts + arguments("a reworded read-only database", 405, + "{\"reason\":\"Database is read only\"}", 1), arguments("maas-agent unable to reach maas-service", 500, "{\"error\":\"error proxying request: connection refused\"}", 2), arguments("throttling", 429, "{\"error\":\"slow down\"}", 1) @@ -78,10 +83,14 @@ static Stream permanentResponses() { // 405 is transient only for a read-only database; a route removed on the server // or an ingress rejecting the method is not arguments("405 without a maas-service envelope", 405, "Method Not Allowed"), - // every maas-service error carries MAAS-0600, so the envelope alone means nothing: - // the reason has to name the read-only database, not merely contain its words + // every maas-service error carries the same code, so the envelope alone means + // nothing: the reason has to name a database that cannot be written arguments("405 whose maas-service reason is unrelated", 405, - "{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}") + "{\"code\":\"MAAS-0600\",\"reason\":\"topic 'active-orders' is inactive\"}"), + arguments("405 about a read-only field rather than the database", 405, + "{\"reason\":\"the read-only field cannot be updated\"}"), + arguments("405 with the marker outside the reason", 405, + "{\"message\":\"database is in read-only mode\"}") ); } diff --git a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java index db7d3d88bb..61b6144bdc 100644 --- a/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java +++ b/maas-client/client/src/test/java/com/netcracker/cloud/maas/client/impl/kafka/KafkaMaaSClientImplTest.java @@ -44,6 +44,41 @@ @Slf4j // TODO tests with kafka SSL+password class KafkaMaaSClientImplTest { + /** maas-agent's answer for the `orders' topic, shared by the tests that just need a valid one. */ + private static final String ORDERS_TOPIC_RESPONSE = """ + { + "addresses": { + "PLAINTEXT": [ + "my-kafka.kafka-cluster:9092" + ] + }, + "name": "maas.core_dev.orders.1234567", + "classifier": { + "name": "orders", + "namespace": "core-dev", + "tenantId": "d047619f-6886-4842-81a7-3f87cb748ac1" + }, + "namespace": "core-dev", + "instance": "default", + "requestedSettings": { + "numPartitions": 1, + "replicationFactor": 1, + "replicaAssignment": null, + "configs": null + }, + "actualSettings": { + "numPartitions": 1, + "replicationFactor": 1, + "replicaAssignment": { + "0": [ 0 ] + }, + "configs": { + "cleanup.policy": "delete" + } + } + } + """; + @BeforeEach public void setup(ClientAndServer mockServer) { mockServer.reset(); @@ -103,37 +138,7 @@ public void testGetTenantTopic(ClientAndServer mockServer) { ).respond( response() .withStatusCode(200) - .withBody("{\n" + - " \"addresses\": {\n" + - " \"PLAINTEXT\": [\n" + - " \"my-kafka.kafka-cluster:9092\"\n" + - " ]\n" + - " }, \n" + - " \"name\": \"maas.core_dev.orders.1234567\",\n" + - " \"classifier\": {\n" + - " \"name\": \"orders\",\n" + - " \"namespace\": \"core-dev\",\n" + - " \"tenantId\": \"d047619f-6886-4842-81a7-3f87cb748ac1\"\n" + - " }, \n" + - " \"namespace\": \"core-dev\",\n" + - " \"instance\": \"default\",\n" + - " \"requestedSettings\": {\n" + - " \"numPartitions\": 1,\n" + - " \"replicationFactor\": 1,\n" + - " \"replicaAssignment\": null,\n" + - " \"configs\": null\n" + - " },\n" + - " \"actualSettings\": {\n" + - " \"numPartitions\": 1,\n" + - " \"replicationFactor\": 1,\n" + - " \"replicaAssignment\": {\n" + - " \"0\": [ 0 ]\n" + - " },\n" + - " \"configs\": {\n" + - " \"cleanup.policy\": \"delete\"\n" + - " }\n" + - " } \n" + - "}\n") + .withBody(ORDERS_TOPIC_RESPONSE) ); // run test @@ -246,37 +251,7 @@ public void testNoNullValuesInCache(ClientAndServer mockServer) { ).respond( response() .withStatusCode(200) - .withBody("{\n" + - " \"addresses\": {\n" + - " \"PLAINTEXT\": [\n" + - " \"my-kafka.kafka-cluster:9092\"\n" + - " ]\n" + - " }, \n" + - " \"name\": \"maas.core_dev.orders.1234567\",\n" + - " \"classifier\": {\n" + - " \"name\": \"orders\",\n" + - " \"namespace\": \"core-dev\",\n" + - " \"tenantId\": \"d047619f-6886-4842-81a7-3f87cb748ac1\"\n" + - " }, \n" + - " \"namespace\": \"core-dev\",\n" + - " \"instance\": \"default\",\n" + - " \"requestedSettings\": {\n" + - " \"numPartitions\": 1,\n" + - " \"replicationFactor\": 1,\n" + - " \"replicaAssignment\": null,\n" + - " \"configs\": null\n" + - " },\n" + - " \"actualSettings\": {\n" + - " \"numPartitions\": 1,\n" + - " \"replicationFactor\": 1,\n" + - " \"replicaAssignment\": {\n" + - " \"0\": [ 0 ]\n" + - " },\n" + - " \"configs\": {\n" + - " \"cleanup.policy\": \"delete\"\n" + - " }\n" + - " } \n" + - "}\n") + .withBody(ORDERS_TOPIC_RESPONSE) ); topicAddress = client.getTopic(new Classifier("orders").tenantId("d047619f-6886-4842-81a7-3f87cb748ac1")); @@ -304,37 +279,7 @@ public void testGetOrCreateLazyTenantTopic(ClientAndServer mockServer) { ).respond( response() .withStatusCode(200) - .withBody("{\n" + - " \"addresses\": {\n" + - " \"PLAINTEXT\": [\n" + - " \"my-kafka.kafka-cluster:9092\"\n" + - " ]\n" + - " }, \n" + - " \"name\": \"maas.core_dev.orders.1234567\",\n" + - " \"classifier\": {\n" + - " \"name\": \"orders\",\n" + - " \"namespace\": \"core-dev\",\n" + - " \"tenantId\": \"d047619f-6886-4842-81a7-3f87cb748ac1\"\n" + - " }, \n" + - " \"namespace\": \"core-dev\",\n" + - " \"instance\": \"default\",\n" + - " \"requestedSettings\": {\n" + - " \"numPartitions\": 1,\n" + - " \"replicationFactor\": 1,\n" + - " \"replicaAssignment\": null,\n" + - " \"configs\": null\n" + - " },\n" + - " \"actualSettings\": {\n" + - " \"numPartitions\": 1,\n" + - " \"replicationFactor\": 1,\n" + - " \"replicaAssignment\": {\n" + - " \"0\": [ 0 ]\n" + - " },\n" + - " \"configs\": {\n" + - " \"cleanup.policy\": \"delete\"\n" + - " }\n" + - " } \n" + - "}\n") + .withBody(ORDERS_TOPIC_RESPONSE) ); // run test