From c0aa47eb6ed5d689479ffbe82a734d9b73386a13 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:09:13 +0900 Subject: [PATCH 1/2] Add optional gzip request compression --- CHANGELOG.md | 1 + sdk-java/api/sdk-java.txt | 4 + .../main/java/ly/count/sdk/java/Config.java | 22 + .../java/internal/ConfigurationProvider.java | 19 + .../java/internal/ImmediateRequestMaker.java | 5 + .../sdk/java/internal/InternalConfig.java | 9 + .../java/internal/ModuleConfiguration.java | 53 +- .../ly/count/sdk/java/internal/Transport.java | 77 +- .../ly/count/sdk/java/internal/Utils.java | 31 + .../internal/ModuleConfigurationTests.java | 2 +- .../internal/RequestCompressionTests.java | 671 ++++++++++++++++++ 11 files changed, 884 insertions(+), 10 deletions(-) create mode 100644 sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a41fa94..212ebd3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ * Added "disableSdkBehaviorSettingsUpdates()" init config method for disabling SDK behavior settings requests. * Improved request queue handling with a built-in backoff mechanism which is enabled by default. When the server takes 10 seconds or longer to answer while few requests are queued, the SDK pauses sending for 60 seconds. * Added "disableBackoffMechanism()" init config method to disable the backoff behavior. +* Added "enableRequestCompression()" init config method. Request bodies between 1 KB and 1 MB are then sent gzip-compressed, once the server's SDK behavior settings answer says it reads them. It does not apply in backend mode, which does not use the SDK behavior settings. * Added support for server requested SDK log gathering. When the server asks for it, the SDK uploads its own internal log lines, even if logging is disabled. While consent is required, a line is uploaded only while the consent of the feature it belongs to is given. * Added support for the SDK connection test. When the server asks for it, the SDK checks that it can reach every endpoint it depends on and reports the results. * Added "setMaxKeyLength(int)" init config method to limit the length of event keys, view names, segmentation keys and custom user property keys. Longer keys are truncated. There is no limit by default, and the key length limit of the SDK behavior settings overrides it. diff --git a/sdk-java/api/sdk-java.txt b/sdk-java/api/sdk-java.txt index 3a6c1a0e..2915e98b 100644 --- a/sdk-java/api/sdk-java.txt +++ b/sdk-java/api/sdk-java.txt @@ -23,6 +23,7 @@ package ly.count.sdk.java { method public ly.count.sdk.java.Config! enableParameterTamperingProtection(String!); method public ly.count.sdk.java.Config! enableRemoteConfigAutomaticTriggers(); method public ly.count.sdk.java.Config! enableRemoteConfigValueCaching(); + method public ly.count.sdk.java.Config! enableRequestCompression(); method @Deprecated public ly.count.sdk.java.Config! enableTestMode(); method @Deprecated public ly.count.sdk.java.Config! enableUsePOST(); method public ly.count.sdk.java.Config! enrollABOnRCDownload(); @@ -131,6 +132,7 @@ package ly.count.sdk.java { field protected int networkRequestCooldown; field protected Set! publicKeyPins; field protected List! remoteConfigGlobalCallbacks; + field protected boolean requestCompressionEnabled; field protected int requestQueueMaxSize; field protected boolean requiresConsent; field protected String! salt; @@ -407,6 +409,7 @@ package ly.count.sdk.java.internal { } public interface ConfigurationProvider { + method public default void blockRequestCompression(); method public boolean getAutomaticCrashReportingEnabled(); method public boolean getAutomaticSessionTrackingEnabled(); method public boolean getAutomaticViewTrackingEnabled(); @@ -439,6 +442,7 @@ package ly.count.sdk.java.internal { method public int getMaxValueSize(); method public boolean getNetworkingEnabled(); method public boolean getRefreshContentZoneEnabled(); + method public default boolean getRequestCompressionEnabled(); method public int getRequestDropAgeHours(); method public int getRequestQueueMaxSize(); method public ly.count.sdk.java.internal.ConfigurationProvider.FilterList!> getSegmentationFilterList(); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/Config.java b/sdk-java/src/main/java/ly/count/sdk/java/Config.java index f49dac67..da5a8242 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/Config.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/Config.java @@ -272,6 +272,12 @@ public class Config { */ protected boolean sdkBehaviorSettingsRequestsDisabled = false; + /** + * Whether request bodies are sent gzip-compressed, once the server's SDK behavior settings + * answer says it reads them. + */ + protected boolean requestCompressionEnabled = false; + /** * Whether the backoff mechanism may hold the request queue back after the server answered * slowly. The SDK behavior settings can override it. @@ -1588,6 +1594,22 @@ public Config disableSdkBehaviorSettingsUpdates() { return this; } + /** + * Compress request bodies between 1 KB and 1 MB with gzip before sending them. The SDK + * compresses only after the server's SDK behavior settings answer said it reads compressed + * bodies, so older servers keep getting plain requests. It needs the SDK behavior settings + * updates, which backend mode and {@link #disableSdkBehaviorSettingsUpdates()} turn off. A + * compressed request that the server, or something in front of it, answers without accepting + * it stops compression until the next SDK behavior settings answer says again that the server + * reads compressed bodies. + * + * @return {@code this} instance for method chaining + */ + public Config enableRequestCompression() { + this.requestCompressionEnabled = true; + return this; + } + /** * Disable the backoff mechanism. While it is on, a request the server takes long to answer * makes the SDK wait before it sends the next queued request, which gives a busy server room diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java index 43356e0d..6a6ad2db 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java @@ -297,6 +297,25 @@ public interface ConfigurationProvider { */ @Nonnull Set getJourneyTriggerViews(); + /** + * Whether a request body may be sent gzip-compressed now: the developer turned request + * compression on, the current server said in its SDK behavior settings answer that it reads + * compressed bodies, and no compressed request was refused since that answer. + * + * @return {@code true} when a request body may be compressed + */ + default boolean getRequestCompressionEnabled() { + return false; + } + + /** + * Stops request compression until the next SDK behavior settings answer says that the server + * reads compressed bodies, or the SDK starts again; called after the server, or something in + * front of it, answered a compressed request without accepting it. + */ + default void blockRequestCompression() { + } + /** * The decision of the {@code lg} directive, which sits next to {@code c} in a live response. * diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java index 66a8326a..7d153afd 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.HttpURLConnection; import java.util.concurrent.CompletableFuture; +import org.json.JSONException; import org.json.JSONObject; /** @@ -93,8 +94,12 @@ private JSONObject doInBackground(String requestData, String customEndpoint, Tra return new JSONObject(receivedBuffer); } else { L.d("[ImmediateRequestMaker] Encountered problem while making a immediate server request, :[" + receivedBuffer + "]"); + cp.blockCompressionIfCompressed(connection); return null; } + } catch (JSONException | StringIndexOutOfBoundsException e) { + L.d("[ImmediateRequestMaker] Received an answer that is empty or not JSON while making a immediate server request, [" + e + "]"); + cp.blockCompressionIfCompressed(connection); } catch (Exception e) { L.d("[ImmediateRequestMaker] Received exception while making a immediate server request " + e.getMessage()); } finally { diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java index 004afb9a..721ccf64 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/InternalConfig.java @@ -184,6 +184,15 @@ public boolean isSdkBehaviorSettingsRequestsDisabled() { return sdkBehaviorSettingsRequestsDisabled; } + /** + * Whether the developer turned request compression on, see {@link Config#enableRequestCompression()}. + * + * @return {@code true} after {@link Config#enableRequestCompression()} + */ + boolean isRequestCompressionEnabled() { + return requestCompressionEnabled; + } + /** * Whether the SDK sends its health check, see {@link Config#disableHealthCheck()}. * diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java index a5a212fc..2ac51835 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java @@ -8,6 +8,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import javax.annotation.Nullable; import ly.count.sdk.java.Config; @@ -38,6 +40,8 @@ public class ModuleConfiguration extends ModuleBase implements ConfigurationProv static final String keyRConfig = "c"; static final String keyRLogGathering = "lg"; static final String keyRConnectionTest = "ct"; + //top level key, a sibling of 'c', read from a live response and never stored + static final String keyRGzip = "gzip"; static final String keyRTracking = "tracking"; static final String keyRNetworking = "networking"; @@ -112,6 +116,10 @@ public class ModuleConfiguration extends ModuleBase implements ConfigurationProv private boolean serverConfigRequestsDisabled = false; private volatile boolean running = false; + + private boolean requestCompressionEnabled = false; + final AtomicReference gzipAcceptingServerUrl = new AtomicReference<>(null); + final AtomicBoolean requestCompressionBlocked = new AtomicBoolean(false); /** * How many times the device ID changed without merge, written under this module's lock, so a * response requested before such a change can tell its gather was armed for the previous user. @@ -230,6 +238,7 @@ public void init(InternalConfig config) { super.init(config); boolean backendMode = config.isBackendModeEnabled(); serverConfigRequestsDisabled = backendMode || config.isSdkBehaviorSettingsRequestsDisabled(); + requestCompressionEnabled = config.isRequestCompressionEnabled(); L.d("[ModuleConfiguration] init, backend mode:[" + backendMode + "], server config requests disabled:[" + serverConfigRequestsDisabled + "]"); seedFromDeveloperConfig(config); @@ -899,7 +908,11 @@ void fetchConfigFromServer(@Nonnull InternalConfig config) { } try { - String requestData = ModuleRequests.prepareRequiredParams(config).add("method", "sc").toString(); + Params params = ModuleRequests.prepareRequiredParams(config).add("method", "sc"); + if (requestCompressionEnabled) { + params.add("gzip", "1"); + } + String requestData = params.toString(); Transport transport = config.sdk.networking.getTransport(); final long fetchStartNs = System.nanoTime(); final int requestedAfterDeviceIdChanges = deviceIdChangesWithoutMerge; @@ -946,6 +959,14 @@ private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JS L.d("[ModuleConfiguration] onServerConfigResponse, retrieved configuration response: [" + response + "]"); long fetchLatencyMs = (System.nanoTime() - fetchStartNs) / 1_000_000L; boolean connectionTestRequested = extractConnectionTestFlag(response); + boolean gzipAccepted = extractGzipFlag(response); + if (requestCompressionEnabled) { + gzipAcceptingServerUrl.set(gzipAccepted ? requestedServerUrl : null); + if (gzipAccepted && requestCompressionBlocked.compareAndSet(true, false)) { + L.i("[ModuleConfiguration] onServerConfigResponse, a behavior settings answer says that the server reads compressed bodies, requests are compressed again"); + } + L.d("[ModuleConfiguration] onServerConfigResponse, the server " + (gzipAccepted ? "reads" : "does not say it reads") + " compressed request bodies"); + } boolean changed; synchronized (this) { @@ -967,6 +988,21 @@ private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JS } } + /** + * Reads and removes the {@code gzip} flag of a live response, so it is never stored: only an + * answer that carries {@code true} says that the server reads compressed request bodies. + * + * @param serverConfigResponse the live response, changed in place + * @return {@code true} only when the response had {@code "gzip": true} + */ + static boolean extractGzipFlag(@Nullable JSONObject serverConfigResponse) { + if (serverConfigResponse == null || !serverConfigResponse.has(keyRGzip)) { + return false; + } + + return Boolean.TRUE.equals(serverConfigResponse.remove(keyRGzip)); + } + /** * Reads and removes the {@code ct} flag of a live response, so it is never stored and can never * come back from storage. Booleans and numbers count by value; a string counts unless it is @@ -1060,6 +1096,21 @@ private synchronized void stopServerConfigUpdateTimer() { } /** {@inheritDoc} */ + @Override + public boolean getRequestCompressionEnabled() { + String acceptingUrl = gzipAcceptingServerUrl.get(); + InternalConfig config = internalConfig; + return requestCompressionEnabled && !requestCompressionBlocked.get() && acceptingUrl != null && config != null + && acceptingUrl.equals(config.getServerURL().toString()); + } + + @Override + public void blockRequestCompression() { + if (requestCompressionBlocked.compareAndSet(false, true)) { + L.i("[ModuleConfiguration] blockRequestCompression, a compressed request was refused, requests are sent uncompressed until a behavior settings answer says that the server reads compressed bodies, or the SDK starts again"); + } + } + @Override public boolean getNetworkingEnabled() { return currentVNetworking; diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java index 4bcfd1e4..a81ee4aa 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java @@ -64,6 +64,7 @@ public class Transport implements X509TrustManager { private Log L = null; private static final String PARAMETER_TAMPERING_DIGEST = "SHA-256"; private static final String CHECKSUM = "checksum256"; + static final String REQUEST_COMPRESSION_MARKER = "content_encoding=gzip"; private InternalConfig config; private SSLContext sslContext; // ssl context to use if pinning is enabled @@ -150,6 +151,22 @@ HttpURLConnection connection(final Request request) throws IOException { request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L)); } + byte[] compressedBody = new byte[0]; + if (!usingGET && maybePictureData == null) { + if (config.getParameterTamperingProtectionSalt() != null) { + request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L)); + } + String markedPath = compressedRequestPath(path); + if (markedPath != null && config.getConfigurationProvider().getRequestCompressionEnabled()) { + String body = request.params.toString(); + compressedBody = Utils.gzipRequestBody(body, L); + if (compressedBody.length > 0) { + path = markedPath; + L.d("[Transport] connection, the request body is sent gzip-compressed, [" + body.length() + "] -> [" + compressedBody.length + "] bytes"); + } + } + } + HttpURLConnection connection = openConnection(path, request.params.toString(), usingGET); connection.setConnectTimeout(1000 * config.getNetworkConnectionTimeout()); connection.setReadTimeout(1000 * config.getNetworkReadTimeout()); @@ -191,17 +208,19 @@ HttpURLConnection connection(final Request request) throws IOException { writer.append("--").append(boundary).append("--").append(Utils.CRLF).flush(); } else { //picture data is "null". If it was sent, we send "null" to server to clear the image there - //we send a normal request in HTTP POST - if (config.getParameterTamperingProtectionSalt() != null) { - request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L)); - } + //we send a normal request in HTTP POST, its checksum was added above connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); output = connection.getOutputStream(); - writer = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true); + if (compressedBody.length > 0) { + output.write(compressedBody); + output.flush(); + } else { + writer = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true); - writer.write(request.params.toString()); - writer.flush(); + writer.write(request.params.toString()); + writer.flush(); + } } } finally { if (writer != null) { @@ -222,6 +241,44 @@ HttpURLConnection connection(final Request request) throws IOException { return connection; } + /** + * The URL a compressed request goes to: the request URL with the compression marker as its only + * query. Endpoints end with an empty query, as in {@code /i?}, which the marker fills. + * + * @param url the URL the request goes to + * @return the URL with the marker, or {@code null} when the URL already has a query, where the + * marker could not be the only one + */ + static String compressedRequestPath(@Nonnull String url) { + int query = url.indexOf('?'); + if (query == -1) { + return url + "?" + REQUEST_COMPRESSION_MARKER; + } + return query == url.length() - 1 ? url + REQUEST_COMPRESSION_MARKER : null; + } + + /** + * Tells whether a connection made by {@link #connection(Request)} sends its body gzip-compressed. + * + * @param connection the connection + * @return {@code true} when its URL carries the compression marker as its query + */ + static boolean isCompressedRequest(@Nullable HttpURLConnection connection) { + return connection != null && REQUEST_COMPRESSION_MARKER.equals(connection.getURL().getQuery()); + } + + /** + * Stops request compression when the connection sent a compressed body; called after its answer + * was not accepted. + * + * @param connection the connection whose answer was not accepted + */ + void blockCompressionIfCompressed(@Nullable HttpURLConnection connection) { + if (isCompressedRequest(connection)) { + config.getConfigurationProvider().blockRequestCompression(); + } + } + /** * Opens the GET of one connection test probe: a bare request, built as * {@link #openBareGetConnection(String, int)} builds it, that verifies the server against the @@ -444,7 +501,11 @@ public Boolean send() { SDKCore.instance.onRequestCompleted(request, response, code, requestOwner); - return processResponse(code, response, request.storageId()); + Boolean accepted = processResponse(code, response, request.storageId()); + if (!Boolean.TRUE.equals(accepted)) { + blockCompressionIfCompressed(connection); + } + return accepted; } catch (IOException e) { L.d("[network] Error while sending request " + request + " " + e); notifyOwnerOfAttemptWithoutResponse(request, requestOwner); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java index b7ac34c8..51b4785b 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Utils.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; /** * Utility class @@ -31,6 +32,9 @@ public class Utils { public static final String UTF8 = "UTF-8"; public static final String CRLF = "\r\n"; public static final char[] BASE_16 = "0123456789ABCDEF".toCharArray(); + static final int REQUEST_COMPRESSION_MIN_BYTES = 1024; + // the server refuses a compressed body that is larger than this once decoded + static final int REQUEST_COMPRESSION_MAX_BYTES = 1024 * 1024; /** * Joins objects with a separator @@ -183,6 +187,33 @@ public static String digestHex(final String digestName, final String string, Log } } + /** + * Compresses a request body with gzip, for sending it with the compression marker as the URL query. + * + * @param body the url encoded request body, checksum included + * @param L logger + * @return the gzip bytes, or an empty array when the body is smaller than + * {@link #REQUEST_COMPRESSION_MIN_BYTES} or larger than {@link #REQUEST_COMPRESSION_MAX_BYTES}, + * compressing fails, or the result is not smaller than the body + */ + static byte[] gzipRequestBody(String body, Log L) { + byte[] plain = body.getBytes(StandardCharsets.UTF_8); + if (plain.length < REQUEST_COMPRESSION_MIN_BYTES || plain.length > REQUEST_COMPRESSION_MAX_BYTES) { + return new byte[0]; + } + + ByteArrayOutputStream out = new ByteArrayOutputStream(plain.length / 2); + try (GZIPOutputStream gzip = new GZIPOutputStream(out)) { + gzip.write(plain); + } catch (IOException e) { + L.d("[Utils] gzipRequestBody, compressing failed, the body is sent uncompressed: [" + e + "]"); + return new byte[0]; + } + + byte[] compressed = out.toByteArray(); + return compressed.length < plain.length ? compressed : new byte[0]; + } + /** * Get hexadecimal string representation of a byte array * diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java index 2e69a4ef..318abc3c 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/ModuleConfigurationTests.java @@ -44,7 +44,7 @@ @RunWith(JUnit4.class) public class ModuleConfigurationTests { - private static final Set TOP_LEVEL_KEYS = names("v", "t", "c", "lg", "ct"); + private static final Set TOP_LEVEL_KEYS = names("v", "t", "c", "lg", "ct", "gzip"); private ServerConfigResponder server; diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java new file mode 100644 index 00000000..35c450af --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java @@ -0,0 +1,671 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; +import ly.count.sdk.java.Config; +import ly.count.sdk.java.Countly; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Request compression, against HTTP servers on loopback that read a body as gzip the way the + * Countly server does: only when the compression marker is the only query of the URL. Each + * settings answer carries its own {@code lkl} value, so a test can wait until the SDK has read it. + * The assertions are on what the servers received. + */ +@RunWith(JUnit4.class) +public class RequestCompressionTests { + + private static final String SALT = "gzip-salt"; + private static final String MARKER_QUERY = "content_encoding=gzip"; + private static final int BIG = 5000; + + private Server server; + private Server otherServer; + + /** + * Starts every test without SDK files and with a new server. + */ + @Before + public void beforeTest() throws IOException { + TestUtils.createCleanTestState(); + server = new Server(); + } + + /** + * Stops the SDK, deletes its data and stops the servers. + */ + @After + public void afterTest() { + Countly.instance().halt(); + server.stop(); + if (otherServer != null) { + otherServer.stop(); + } + } + + /** + * Without {@link Config#enableRequestCompression()} the settings request does not ask for + * gzip, and a big request goes plain although the server says it reads gzip. + */ + @Test + public void compressionOff_settingsRequestDoesNotAskAndRequestsArePlain() throws Exception { + server.answer(101, true); + Countly.instance().init(config(server)); + awaitSettings(101); + + queueRequest("big", BIG); + Received big = server.await(withMarker("big")); + + Assert.assertNull(server.await(withMethod("sc")).params.get("gzip")); + Assert.assertFalse(big.compressed); + Assert.assertEquals(BIG, big.params.get("payload").length()); + Assert.assertFalse(provider().getRequestCompressionEnabled()); + } + + /** + * Once the server says it reads gzip, a body of at least 1 KB goes compressed to the endpoint + * with the marker as the only query, as a form body without a Content-Encoding header, and its + * checksum is the one of the plain body. + */ + @Test + public void compressionOn_serverReadsGzip_bigRequestIsCompressed() throws Exception { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression().enableParameterTamperingProtection(SALT)); + awaitSettings(101); + Assert.assertTrue(provider().getRequestCompressionEnabled()); + + queueRequest("big", BIG); + Received big = server.await(withMarker("big")); + + Assert.assertEquals("1", server.await(withMethod("sc")).params.get("gzip")); + Assert.assertTrue(big.compressed); + Assert.assertEquals("/i", big.path); + Assert.assertTrue("the compressed body has " + big.bodyBytes + " bytes", big.bodyBytes < 1000); + Assert.assertEquals("application/x-www-form-urlencoded", big.contentType); + Assert.assertNull(big.contentEncoding); + Assert.assertEquals(repeat('a', BIG), big.params.get("payload")); + String[] signed = big.body.split("&checksum256="); + Assert.assertEquals(2, signed.length); + Assert.assertEquals(Utils.digestHex("SHA-256", signed[0] + SALT, null), signed[1]); + } + + /** + * A body under 1 KB goes plain, also as a POST, and a bigger one after it is compressed. + */ + @Test + public void compressionOn_smallRequestStaysPlain() throws Exception { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression().enableForcedHTTPPost()); + awaitSettings(101); + + queueRequest("small", 300); + Received small = server.await(withMarker("small")); + queueRequest("big", BIG); + Received big = server.await(withMarker("big")); + + Assert.assertEquals("POST", small.method); + Assert.assertFalse(small.compressed); + Assert.assertTrue(small.bodyBytes < 1024); + Assert.assertTrue(big.compressed); + } + + /** + * A settings answer without the gzip key keeps every body plain. + */ + @Test + public void compressionOn_serverDoesNotSayGzip_requestsArePlain() throws Exception { + server.answer(101, false); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("big", BIG); + Received big = server.await(withMarker("big")); + + Assert.assertEquals("1", server.await(withMethod("sc")).params.get("gzip")); + Assert.assertFalse(big.compressed); + Assert.assertFalse(provider().getRequestCompressionEnabled()); + } + + /** + * A body over 1 MB goes plain, as the server refuses a bigger compressed body, and compression + * stays on for the requests after it. + */ + @Test + public void compressionOn_bodyOverOneMegabyteStaysPlain_compressionStaysOn() throws Exception { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("huge", Utils.REQUEST_COMPRESSION_MAX_BYTES); + Received huge = server.await(withMarker("huge")); + queueRequest("big", BIG); + Received big = server.await(withMarker("big")); + + Assert.assertFalse(huge.compressed); + Assert.assertTrue(huge.bodyBytes > Utils.REQUEST_COMPRESSION_MAX_BYTES); + Assert.assertTrue(big.compressed); + } + + /** + * A refused compressed request stays queued and goes plain on its next try, and so do the + * requests after it. + */ + @Test + public void compressedRequestRefused_retryAndLaterRequestsArePlain() throws Exception { + server.answer(101, true); + server.refuseCompressed = 1; + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("refused", BIG); + server.await(withMarker("refused")); + awaitSendHandled(); + Countly.instance().attemptToSendStoredRequests(); + Assert.assertTrue(TestUtils.waitFor(15_000, () -> server.count(withMarker("refused")) == 2)); + queueRequest("after", BIG); + Received after = server.await(withMarker("after")); + + List refused = server.all(withMarker("refused")); + Assert.assertEquals(Arrays.asList(true, false), compressedFlags(refused)); + Assert.assertEquals(refused.get(0).params, refused.get(1).params); + Assert.assertFalse(after.compressed); + } + + /** + * After a refusal stopped compression, a settings answer without gzip leaves it stopped, and the + * next one with "gzip": true turns it on again. + */ + @Test + public void compressedRequestRefused_nextGzipAnswerTurnsCompressionOnAgain() throws Exception { + server.answer(101, true); + server.refuseCompressed = 1; + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + queueRequest("refused", BIG); + server.await(withMarker("refused")); + Assert.assertTrue(TestUtils.waitFor(10_000, () -> !provider().getRequestCompressionEnabled())); + + server.answer(102, false); + fetchSettings(); + awaitSettings(102); + Assert.assertFalse(provider().getRequestCompressionEnabled()); + + server.answer(103, true); + fetchSettings(); + awaitSettings(103); + Assert.assertTrue(TestUtils.waitFor(10_000, () -> provider().getRequestCompressionEnabled())); + awaitSendHandled(); + Countly.instance().attemptToSendStoredRequests(); + queueRequest("after_gzip_answer", BIG); + Received after = server.await(withMarker("after_gzip_answer")); + + Assert.assertTrue(after.compressed); + Assert.assertEquals(Arrays.asList(true, false), compressedFlags(server.all(withMarker("refused")))); + } + + /** + * A compressed remote config download answered with JSON leaves compression on. + */ + @Test + public void compressedImmediateRequestAnsweredWithJson_compressionStaysOn() throws Exception { + Assert.assertTrue(immediateRequestKeepsCompression(200, "{\"color\":\"red\"}")); + } + + /** + * A compressed remote config download answered with a page, as a proxy in front of the server + * may send, was not accepted: compression stops. + */ + @Test + public void compressedImmediateRequestAnsweredWithPage_stopsCompression() throws Exception { + Assert.assertFalse(immediateRequestKeepsCompression(200, "Bad gateway")); + } + + /** + * A compressed remote config download answered with an error status was not accepted: + * compression stops. + */ + @Test + public void compressedImmediateRequestAnsweredWithError_stopsCompression() throws Exception { + Assert.assertFalse(immediateRequestKeepsCompression(400, "{\"result\":\"Invalid request body\"}")); + } + + /** + * A compressed remote config download answered with nothing was not accepted: compression stops. + */ + @Test + public void compressedImmediateRequestAnsweredWithNothing_stopsCompression() throws Exception { + Assert.assertFalse(immediateRequestKeepsCompression(200, "")); + } + + /** + * A compressed request that failed without an answer does not stop compression: the next try + * goes compressed again. + */ + @Test + public void compressedRequestWithoutAnswer_compressionStaysOn() throws Exception { + server.answer(101, true); + server.dropCompressed = true; + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + queueRequest("dropped", BIG); + server.await(withMarker("dropped")); + awaitSendHandled(); + server.dropCompressed = false; + Countly.instance().attemptToSendStoredRequests(); + Assert.assertTrue(TestUtils.waitFor(15_000, () -> server.count(withMarker("dropped").and(r -> r.answered)) == 1)); + + List dropped = server.all(withMarker("dropped")); + Assert.assertTrue(dropped.size() >= 2); + for (Received received : dropped) { + Assert.assertTrue(received.compressed); + } + Assert.assertTrue(provider().getRequestCompressionEnabled()); + } + + /** + * The gzip key is read from the live answer only, never stored with the settings. + */ + @Test + public void gzipAnswer_isNotStored() throws Exception { + server.answer(101, true); + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + ModuleConfiguration configuration = SDKCore.instance.module(ModuleConfiguration.class); + synchronized (configuration) { + Assert.assertFalse(configuration.latestRetrievedConfiguration.has("gzip")); + } + String stored = SDKCore.instance.sdkStorage.getServerConfig(); + Assert.assertNotNull(stored); + Assert.assertFalse(stored.contains("gzip")); + } + + /** + * After a server URL change the requests go plain, until the new server says it reads gzip. + */ + @Test + public void serverUrlChange_newServerGetsPlainRequestsUntilItSaysGzip() throws Exception { + server.answer(101, true); + otherServer = new Server(); + otherServer.answer(102, true); + CountDownLatch release = new CountDownLatch(1); + otherServer.holdSettings = release; + Countly.instance().init(config(server).enableRequestCompression()); + awaitSettings(101); + + Countly.instance().changeServerUrl(otherServer.url()); + otherServer.await(withMethod("sc")); + queueRequest("before_answer", BIG); + Received before = otherServer.await(withMarker("before_answer")); + release.countDown(); + awaitSettings(102); + queueRequest("after_answer", BIG); + Received after = otherServer.await(withMarker("after_answer")); + + Assert.assertFalse(before.compressed); + Assert.assertTrue(after.compressed); + Assert.assertEquals(0, server.count(withMarker("before_answer"))); + } + + /** + * Only an answer with "gzip": true counts, and the key is removed from it. + */ + @Test + public void extractGzipFlag_onlyTrueCounts() { + JSONObject accepted = new JSONObject("{\"v\":1,\"t\":1,\"c\":{},\"gzip\":true}"); + Assert.assertTrue(ModuleConfiguration.extractGzipFlag(accepted)); + Assert.assertFalse(accepted.has("gzip")); + + for (String answer : new String[] { "{\"gzip\":false}", "{\"gzip\":\"true\"}", "{\"gzip\":1}", "{\"c\":{\"gzip\":true}}", "{}" }) { + Assert.assertFalse(answer, ModuleConfiguration.extractGzipFlag(new JSONObject(answer))); + } + Assert.assertFalse(ModuleConfiguration.extractGzipFlag(null)); + } + + /** + * Bodies under 1 KB and over 1 MB are left alone, the ones between are compressed into gzip + * that decodes to the body. + */ + @Test + public void gzipRequestBody_bounds() throws IOException { + Log log = new Log(Config.LoggingLevel.OFF, null); + Assert.assertEquals(0, Utils.gzipRequestBody(repeat('a', Utils.REQUEST_COMPRESSION_MIN_BYTES - 1), log).length); + Assert.assertEquals(0, Utils.gzipRequestBody(repeat('a', Utils.REQUEST_COMPRESSION_MAX_BYTES + 1), log).length); + for (int size : new int[] { Utils.REQUEST_COMPRESSION_MIN_BYTES, Utils.REQUEST_COMPRESSION_MAX_BYTES }) { + String body = "app_key=key&payload=" + repeat('a', size - 20); + byte[] compressed = Utils.gzipRequestBody(body, log); + Assert.assertTrue(compressed.length > 0 && compressed.length < body.length()); + Assert.assertEquals(body, new String(gunzip(compressed), StandardCharsets.UTF_8)); + } + } + + /** + * The marker fills the empty query of an endpoint, or becomes the query of a URL without one; a + * URL that already has a query cannot carry it. + */ + @Test + public void compressedRequestPath_markerIsTheOnlyQuery() { + Assert.assertEquals("http://host/i?content_encoding=gzip", Transport.compressedRequestPath("http://host/i?")); + Assert.assertEquals("http://host/o/sdk?content_encoding=gzip", Transport.compressedRequestPath("http://host/o/sdk")); + Assert.assertNull(Transport.compressedRequestPath("http://host/o/sdk?method=rc")); + } + + // region helpers + + /** + * Downloads remote config with 120 keys, a compressed POST, answered with the given status and + * text, then tells whether a big request sent after it was compressed. + */ + private boolean immediateRequestKeepsCompression(int remoteConfigCode, String remoteConfigAnswer) throws Exception { + server.answer(101, true); + server.remoteConfigCode = remoteConfigCode; + server.remoteConfigAnswer = remoteConfigAnswer; + Countly.instance().init(config(server, Config.Feature.RemoteConfig).enableRequestCompression()); + awaitSettings(101); + + String[] keys = new String[120]; + for (int i = 0; i < keys.length; i++) { + keys[i] = "remote_config_key_" + i; + } + CountDownLatch downloaded = new CountDownLatch(1); + Countly.instance().remoteConfig().downloadSpecificKeys(keys, (result, error, fullValueUpdate, values) -> downloaded.countDown()); + Assert.assertTrue(downloaded.await(15, TimeUnit.SECONDS)); + queueRequest("after_download", BIG); + Received after = server.await(withMarker("after_download")); + + Received download = server.await(withMethod("rc")); + Assert.assertTrue(download.compressed); + Assert.assertEquals(120, new org.json.JSONArray(download.params.get("keys")).length()); + return after.compressed; + } + + /** + * A configuration that sends to the given server without request cooldowns or health check. + */ + private static Config config(Server server, Config.Feature... features) { + File directory = TestUtils.getTestSDirectory(); + TestUtils.checkSdkStorageRootDirectoryExist(directory); + + Config config = new Config(server.url(), TestUtils.SERVER_APP_KEY, directory); + config.setApplicationVersion(TestUtils.APPLICATION_VERSION); + config.setCustomDeviceId(TestUtils.DEVICE_ID); + config.disableHealthCheck(); + config.enableFeatures(features); + config.setNetworkRequestCooldown(0); + config.setNetworkImportantRequestCooldown(0); + return config; + } + + /** + * The SDK behavior settings in effect. + */ + private static ConfigurationProvider provider() { + return SDKCore.instance.config.getConfigurationProvider(); + } + + /** + * Waits until the SDK has read the settings answer that carries the given {@code lkl} value. + */ + private static void awaitSettings(int lkl) throws InterruptedException { + Assert.assertTrue("the settings answer with lkl " + lkl + " was not read", TestUtils.waitFor(15_000, () -> provider().getMaxKeyLength() == lkl)); + } + + /** + * Fetches the settings again, as the refresh timer does. + */ + private static void fetchSettings() { + SDKCore.instance.module(ModuleConfiguration.class).fetchConfigFromServer(SDKCore.instance.config); + } + + /** + * Queues a request with a payload of the given length, told apart by its marker parameter. + */ + private static void queueRequest(String marker, int payloadLength) { + InternalConfig config = SDKCore.instance.config; + Request request = new Request(); + request.params.add(ModuleRequests.prepareRequiredParams(config)); + request.params.add("marker", marker); + request.params.add("payload", repeat('a', payloadLength)); + ModuleRequests.pushAsync(config, request); + } + + /** + * Waits until the SDK finished handling the request it is sending, so a send asked for next is + * not folded into it. + */ + private static void awaitSendHandled() throws InterruptedException { + Assert.assertTrue("the send in progress must end", TestUtils.waitFor(10_000, () -> !SDKCore.instance.networking.isSending())); + } + + /** + * Matches the request that carries the given marker parameter. + */ + private static Predicate withMarker(String marker) { + return r -> marker.equals(r.params.get("marker")); + } + + /** + * Matches an immediate request of the given method. + */ + private static Predicate withMethod(String method) { + return r -> method.equals(r.params.get("method")); + } + + /** + * Whether each request came compressed. + */ + private static List compressedFlags(List requests) { + return requests.stream().map(r -> r.compressed).collect(Collectors.toList()); + } + + /** + * A text of the given length made of one character. + */ + private static String repeat(char c, int length) { + char[] chars = new char[length]; + Arrays.fill(chars, c); + return new String(chars); + } + + /** + * Decodes gzip bytes. + */ + private static byte[] gunzip(byte[] compressed) throws IOException { + try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(compressed))) { + return readAll(in); + } + } + + /** + * Reads a stream to its end. + */ + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = stream.read(buffer)) != -1) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + /** + * One request as a server received it: the body decoded and its parameters read the way the + * Countly server reads them. + */ + private static final class Received { + final String method; + final String path; + final boolean compressed; + final String body; + final int bodyBytes; + final String contentType; + final String contentEncoding; + final Map params; + volatile boolean answered = false; + + /** + * Reads a request from its exchange. + */ + Received(HttpExchange exchange) throws IOException { + method = exchange.getRequestMethod(); + path = exchange.getRequestURI().getPath(); + String query = exchange.getRequestURI().getRawQuery(); + compressed = MARKER_QUERY.equals(query); + byte[] raw = readAll(exchange.getRequestBody()); + bodyBytes = raw.length; + body = new String(compressed ? gunzip(raw) : raw, StandardCharsets.UTF_8); + contentType = exchange.getRequestHeaders().getFirst("Content-Type"); + contentEncoding = exchange.getRequestHeaders().getFirst("Content-Encoding"); + params = TestUtils.parseQueryParams(compressed || !Utils.isNotEmpty(query) ? body : query); + params.replaceAll((key, value) -> Utils.urldecode(value)); + } + } + + /** + * A Countly server stand-in on loopback that records every request. It answers the SDK behavior + * settings with the answer set by {@link #answer(int, boolean)}, remote config with + * {@link #remoteConfigCode} and {@link #remoteConfigAnswer}, and anything else with success; it can refuse or drop compressed + * requests and hold its settings answer until a latch opens. + */ + private static final class Server { + final HttpServer http; + final List received = new CopyOnWriteArrayList<>(); + volatile String settingsAnswer = "{}"; + volatile String remoteConfigAnswer = "{}"; + volatile int remoteConfigCode = 200; + volatile int refuseCompressed = 0; + volatile boolean dropCompressed = false; + volatile CountDownLatch holdSettings = null; + + /** + * Starts the server on a free loopback port. + */ + Server() throws IOException { + http = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + http.createContext("/", this::handle); + http.setExecutor(Executors.newCachedThreadPool()); + http.start(); + } + + /** + * Sets the settings answer: an {@code lkl} value that tells it apart, with "gzip": true when + * the server reads compressed bodies. + */ + void answer(int lkl, boolean gzip) { + JSONObject answer = new JSONObject().put("v", 1).put("t", 1_700_000_000_000L).put("c", new JSONObject().put("lkl", lkl)); + if (gzip) { + answer.put("gzip", true); + } + settingsAnswer = answer.toString(); + } + + /** + * Records a request and answers it. + */ + private void handle(HttpExchange exchange) throws IOException { + Received request = new Received(exchange); + received.add(request); + String method = request.params.get("method"); + int code = 200; + String response = "{\"result\":\"Success\"}"; + if ("sc".equals(method)) { + CountDownLatch hold = holdSettings; + if (hold != null) { + try { + hold.await(15, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + response = settingsAnswer; + } else if ("rc".equals(method)) { + code = remoteConfigCode; + response = remoteConfigAnswer; + } + if (request.compressed && dropCompressed) { + exchange.close(); + return; + } + if (request.compressed && refuseCompressed > 0) { + refuseCompressed--; + code = 400; + response = "{\"result\":\"Invalid request body\"}"; + } + byte[] out = response.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(code, out.length == 0 ? -1 : out.length); + try (OutputStream stream = exchange.getResponseBody()) { + stream.write(out); + } + request.answered = true; + } + + /** + * The URL the SDK reaches this server at. + */ + String url() { + return "http://127.0.0.1:" + http.getAddress().getPort(); + } + + /** + * How many received requests match. + */ + int count(Predicate filter) { + return (int) received.stream().filter(filter).count(); + } + + /** + * Every received request that matches, in arrival order. + */ + List all(Predicate filter) { + return new ArrayList<>(received.stream().filter(filter).collect(Collectors.toList())); + } + + /** + * Waits for the first matching request and returns it. + */ + Received await(Predicate filter) throws InterruptedException { + Assert.assertTrue("no matching request arrived at " + url() + ", it received " + received.size(), TestUtils.waitFor(15_000, () -> count(filter) > 0)); + return received.stream().filter(filter).findFirst().orElse(null); + } + + /** + * Opens a held settings answer and stops the server. + */ + void stop() { + CountDownLatch hold = holdSettings; + if (hold != null) { + hold.countDown(); + } + http.stop(0); + } + } + + // endregion +} From 953904022e053f2de20f326d18697c459bc3cf18 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:25:42 +0900 Subject: [PATCH 2/2] Fix the static analysis findings of request compression --- sdk-java/api/sdk-java.txt | 2 +- .../java/internal/ConfigurationProvider.java | 2 +- .../java/internal/ImmediateRequestMaker.java | 2 +- .../java/internal/ModuleConfiguration.java | 12 +- .../ly/count/sdk/java/internal/Transport.java | 2 +- .../sdk/java/internal/GzipStandInRequest.java | 82 +++++ .../sdk/java/internal/GzipStandInServer.java | 161 ++++++++++ .../internal/RequestCompressionTests.java | 297 ++++-------------- 8 files changed, 315 insertions(+), 245 deletions(-) create mode 100644 sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInRequest.java create mode 100644 sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInServer.java diff --git a/sdk-java/api/sdk-java.txt b/sdk-java/api/sdk-java.txt index 2915e98b..0aec5f02 100644 --- a/sdk-java/api/sdk-java.txt +++ b/sdk-java/api/sdk-java.txt @@ -442,7 +442,6 @@ package ly.count.sdk.java.internal { method public int getMaxValueSize(); method public boolean getNetworkingEnabled(); method public boolean getRefreshContentZoneEnabled(); - method public default boolean getRequestCompressionEnabled(); method public int getRequestDropAgeHours(); method public int getRequestQueueMaxSize(); method public ly.count.sdk.java.internal.ConfigurationProvider.FilterList!> getSegmentationFilterList(); @@ -453,6 +452,7 @@ package ly.count.sdk.java.internal { method public int getUserPropertyCacheLimit(); method public ly.count.sdk.java.internal.ConfigurationProvider.FilterList!> getUserPropertyFilterList(); method public boolean getViewTrackingEnabled(); + method public default boolean isRequestCompressionAllowed(); method public boolean isRequestQueueMaxSizeFromBehaviorSettings(); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java index 6a6ad2db..9845a302 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ConfigurationProvider.java @@ -304,7 +304,7 @@ public interface ConfigurationProvider { * * @return {@code true} when a request body may be compressed */ - default boolean getRequestCompressionEnabled() { + default boolean isRequestCompressionAllowed() { return false; } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java index 7d153afd..e9f065f2 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ImmediateRequestMaker.java @@ -98,7 +98,7 @@ private JSONObject doInBackground(String requestData, String customEndpoint, Tra return null; } } catch (JSONException | StringIndexOutOfBoundsException e) { - L.d("[ImmediateRequestMaker] Received an answer that is empty or not JSON while making a immediate server request, [" + e + "]"); + L.d("[ImmediateRequestMaker] Received an answer that is empty or not JSON while making a immediate server request, [" + e.getMessage() + "]"); cp.blockCompressionIfCompressed(connection); } catch (Exception e) { L.d("[ImmediateRequestMaker] Received exception while making a immediate server request " + e.getMessage()); diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java index 2ac51835..0bfd5520 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/ModuleConfiguration.java @@ -117,7 +117,7 @@ public class ModuleConfiguration extends ModuleBase implements ConfigurationProv private boolean serverConfigRequestsDisabled = false; private volatile boolean running = false; - private boolean requestCompressionEnabled = false; + private final AtomicBoolean requestCompressionEnabled = new AtomicBoolean(false); final AtomicReference gzipAcceptingServerUrl = new AtomicReference<>(null); final AtomicBoolean requestCompressionBlocked = new AtomicBoolean(false); /** @@ -238,7 +238,7 @@ public void init(InternalConfig config) { super.init(config); boolean backendMode = config.isBackendModeEnabled(); serverConfigRequestsDisabled = backendMode || config.isSdkBehaviorSettingsRequestsDisabled(); - requestCompressionEnabled = config.isRequestCompressionEnabled(); + requestCompressionEnabled.set(config.isRequestCompressionEnabled()); L.d("[ModuleConfiguration] init, backend mode:[" + backendMode + "], server config requests disabled:[" + serverConfigRequestsDisabled + "]"); seedFromDeveloperConfig(config); @@ -909,7 +909,7 @@ void fetchConfigFromServer(@Nonnull InternalConfig config) { try { Params params = ModuleRequests.prepareRequiredParams(config).add("method", "sc"); - if (requestCompressionEnabled) { + if (requestCompressionEnabled.get()) { params.add("gzip", "1"); } String requestData = params.toString(); @@ -960,7 +960,7 @@ private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JS long fetchLatencyMs = (System.nanoTime() - fetchStartNs) / 1_000_000L; boolean connectionTestRequested = extractConnectionTestFlag(response); boolean gzipAccepted = extractGzipFlag(response); - if (requestCompressionEnabled) { + if (requestCompressionEnabled.get()) { gzipAcceptingServerUrl.set(gzipAccepted ? requestedServerUrl : null); if (gzipAccepted && requestCompressionBlocked.compareAndSet(true, false)) { L.i("[ModuleConfiguration] onServerConfigResponse, a behavior settings answer says that the server reads compressed bodies, requests are compressed again"); @@ -1097,10 +1097,10 @@ private synchronized void stopServerConfigUpdateTimer() { /** {@inheritDoc} */ @Override - public boolean getRequestCompressionEnabled() { + public boolean isRequestCompressionAllowed() { String acceptingUrl = gzipAcceptingServerUrl.get(); InternalConfig config = internalConfig; - return requestCompressionEnabled && !requestCompressionBlocked.get() && acceptingUrl != null && config != null + return requestCompressionEnabled.get() && !requestCompressionBlocked.get() && acceptingUrl != null && config != null && acceptingUrl.equals(config.getServerURL().toString()); } diff --git a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java index a81ee4aa..3bd05c6e 100644 --- a/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java +++ b/sdk-java/src/main/java/ly/count/sdk/java/internal/Transport.java @@ -157,7 +157,7 @@ HttpURLConnection connection(final Request request) throws IOException { request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L)); } String markedPath = compressedRequestPath(path); - if (markedPath != null && config.getConfigurationProvider().getRequestCompressionEnabled()) { + if (markedPath != null && config.getConfigurationProvider().isRequestCompressionAllowed()) { String body = request.params.toString(); compressedBody = Utils.gzipRequestBody(body, L); if (compressedBody.length > 0) { diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInRequest.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInRequest.java new file mode 100644 index 00000000..2e5595c5 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInRequest.java @@ -0,0 +1,82 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.GZIPInputStream; + +/** + * One request as {@link GzipStandInServer} received it: the body decoded and its parameters read + * the way the Countly server reads them, which takes a body as gzip only when the compression + * marker is the only query of the URL. + */ +final class GzipStandInRequest { + + static final String MARKER_QUERY = "content_encoding=gzip"; + + final String method; + final String path; + final boolean compressed; + final String body; + final int bodyBytes; + final String contentType; + final String contentEncoding; + final Map params; + final AtomicBoolean answered = new AtomicBoolean(false); + + /** + * Reads a request from its exchange. + * + * @param exchange the exchange the request arrived in + * @throws IOException when the body cannot be read or decoded + */ + GzipStandInRequest(HttpExchange exchange) throws IOException { + method = exchange.getRequestMethod(); + path = exchange.getRequestURI().getPath(); + String query = exchange.getRequestURI().getRawQuery(); + compressed = MARKER_QUERY.equals(query); + byte[] raw = readAll(exchange.getRequestBody()); + bodyBytes = raw.length; + body = new String(compressed ? gunzip(raw) : raw, StandardCharsets.UTF_8); + contentType = exchange.getRequestHeaders().getFirst("Content-Type"); + contentEncoding = exchange.getRequestHeaders().getFirst("Content-Encoding"); + params = TestUtils.parseQueryParams(compressed || !Utils.isNotEmpty(query) ? body : query); + params.replaceAll((key, value) -> Utils.urldecode(value)); + } + + /** + * Decodes gzip bytes. + * + * @param compressed gzip bytes + * @return the decoded bytes + * @throws IOException when the bytes are not gzip + */ + static byte[] gunzip(byte[] compressed) throws IOException { + try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(compressed))) { + return readAll(in); + } + } + + /** + * Reads a stream to its end. + * + * @param stream the stream to read + * @return every byte of the stream + * @throws IOException when the stream cannot be read + */ + static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read = stream.read(buffer); + while (read != -1) { + out.write(buffer, 0, read); + read = stream.read(buffer); + } + return out.toByteArray(); + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInServer.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInServer.java new file mode 100644 index 00000000..558ec569 --- /dev/null +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/GzipStandInServer.java @@ -0,0 +1,161 @@ +package ly.count.sdk.java.internal; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.json.JSONObject; +import org.junit.Assert; + +/** + * A Countly server stand-in on loopback for the request compression tests, which records every + * request. It answers the SDK behavior settings with the answer set by {@link #answer(int, boolean)}, + * remote config with {@link #remoteConfigCode} and {@link #remoteConfigAnswer}, and anything else + * with success; it can refuse or drop compressed requests and hold its settings answer until a + * latch opens. + */ +final class GzipStandInServer { + + final HttpServer http; + final List received = new CopyOnWriteArrayList<>(); + final AtomicReference settingsAnswer = new AtomicReference<>("{}"); + final AtomicReference remoteConfigAnswer = new AtomicReference<>("{}"); + final AtomicInteger remoteConfigCode = new AtomicInteger(200); + final AtomicInteger refuseCompressed = new AtomicInteger(0); + final AtomicBoolean dropCompressed = new AtomicBoolean(false); + final AtomicReference holdSettings = new AtomicReference<>(null); + + /** + * Starts the server on a free loopback port. + * + * @throws IOException when the server cannot start + */ + GzipStandInServer() throws IOException { + http = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + http.createContext("/", this::handle); + http.setExecutor(Executors.newCachedThreadPool()); + http.start(); + } + + /** + * Sets the settings answer: an {@code lkl} value that tells it apart, with "gzip": true when + * the server reads compressed bodies. + * + * @param lkl the key length limit of the answer + * @param gzip whether the answer says that the server reads compressed bodies + */ + void answer(int lkl, boolean gzip) { + JSONObject answer = new JSONObject().put("v", 1).put("t", 1_700_000_000_000L).put("c", new JSONObject().put("lkl", lkl)); + if (gzip) { + answer.put("gzip", true); + } + settingsAnswer.set(answer.toString()); + } + + /** + * Records a request and answers it. + * + * @param exchange the exchange the request arrived in + * @throws IOException when the request cannot be read or answered + */ + private void handle(HttpExchange exchange) throws IOException { + GzipStandInRequest request = new GzipStandInRequest(exchange); + received.add(request); + String method = request.params.get("method"); + int code = 200; + String response = "{\"result\":\"Success\"}"; + if ("sc".equals(method)) { + CountDownLatch hold = holdSettings.get(); + if (hold != null) { + try { + hold.await(15, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + response = settingsAnswer.get(); + } else if ("rc".equals(method)) { + code = remoteConfigCode.get(); + response = remoteConfigAnswer.get(); + } + if (request.compressed && dropCompressed.get()) { + exchange.close(); + return; + } + if (request.compressed && refuseCompressed.getAndUpdate(left -> left > 0 ? left - 1 : 0) > 0) { + code = 400; + response = "{\"result\":\"Invalid request body\"}"; + } + byte[] out = response.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(code, out.length == 0 ? -1 : out.length); + try (OutputStream stream = exchange.getResponseBody()) { + stream.write(out); + } + request.answered.set(true); + } + + /** + * The URL the SDK reaches this server at. + * + * @return the server URL + */ + String url() { + return "http://127.0.0.1:" + http.getAddress().getPort(); + } + + /** + * How many received requests match. + * + * @param filter the requests to count + * @return the number of matching requests + */ + int count(Predicate filter) { + return (int) received.stream().filter(filter).count(); + } + + /** + * Every received request that matches, in arrival order. + * + * @param filter the requests to return + * @return the matching requests + */ + List all(Predicate filter) { + return received.stream().filter(filter).collect(Collectors.toList()); + } + + /** + * Waits for the first matching request and returns it. + * + * @param filter the request to wait for + * @return the first matching request + * @throws InterruptedException when the wait is interrupted + */ + GzipStandInRequest await(Predicate filter) throws InterruptedException { + Assert.assertTrue("no matching request arrived at " + url() + ", it received " + received.size(), TestUtils.waitFor(15_000, () -> count(filter) > 0)); + return received.stream().filter(filter).findFirst().orElse(null); + } + + /** + * Opens a held settings answer and stops the server. + */ + void stop() { + CountDownLatch hold = holdSettings.get(); + if (hold != null) { + hold.countDown(); + } + http.stop(0); + } +} diff --git a/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java b/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java index 35c450af..d72dada1 100644 --- a/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java +++ b/sdk-java/src/test/java/ly/count/sdk/java/internal/RequestCompressionTests.java @@ -1,26 +1,14 @@ package ly.count.sdk.java.internal; -import com.sun.net.httpserver.HttpExchange; -import com.sun.net.httpserver.HttpServer; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; import ly.count.sdk.java.Config; import ly.count.sdk.java.Countly; import org.json.JSONObject; @@ -41,11 +29,10 @@ public class RequestCompressionTests { private static final String SALT = "gzip-salt"; - private static final String MARKER_QUERY = "content_encoding=gzip"; private static final int BIG = 5000; - private Server server; - private Server otherServer; + private GzipStandInServer server; + private GzipStandInServer otherServer; /** * Starts every test without SDK files and with a new server. @@ -53,7 +40,7 @@ public class RequestCompressionTests { @Before public void beforeTest() throws IOException { TestUtils.createCleanTestState(); - server = new Server(); + server = new GzipStandInServer(); } /** @@ -73,18 +60,18 @@ public void afterTest() { * gzip, and a big request goes plain although the server says it reads gzip. */ @Test - public void compressionOff_settingsRequestDoesNotAskAndRequestsArePlain() throws Exception { + public void compressionOff_settingsRequestDoesNotAskAndRequestsArePlain() throws InterruptedException { server.answer(101, true); Countly.instance().init(config(server)); awaitSettings(101); queueRequest("big", BIG); - Received big = server.await(withMarker("big")); + GzipStandInRequest big = server.await(withMarker("big")); Assert.assertNull(server.await(withMethod("sc")).params.get("gzip")); Assert.assertFalse(big.compressed); Assert.assertEquals(BIG, big.params.get("payload").length()); - Assert.assertFalse(provider().getRequestCompressionEnabled()); + Assert.assertFalse(provider().isRequestCompressionAllowed()); } /** @@ -93,14 +80,14 @@ public void compressionOff_settingsRequestDoesNotAskAndRequestsArePlain() throws * checksum is the one of the plain body. */ @Test - public void compressionOn_serverReadsGzip_bigRequestIsCompressed() throws Exception { + public void compressionOn_serverReadsGzip_bigRequestIsCompressed() throws InterruptedException { server.answer(101, true); Countly.instance().init(config(server).enableRequestCompression().enableParameterTamperingProtection(SALT)); awaitSettings(101); - Assert.assertTrue(provider().getRequestCompressionEnabled()); + Assert.assertTrue(provider().isRequestCompressionAllowed()); queueRequest("big", BIG); - Received big = server.await(withMarker("big")); + GzipStandInRequest big = server.await(withMarker("big")); Assert.assertEquals("1", server.await(withMethod("sc")).params.get("gzip")); Assert.assertTrue(big.compressed); @@ -118,15 +105,15 @@ public void compressionOn_serverReadsGzip_bigRequestIsCompressed() throws Except * A body under 1 KB goes plain, also as a POST, and a bigger one after it is compressed. */ @Test - public void compressionOn_smallRequestStaysPlain() throws Exception { + public void compressionOn_smallRequestStaysPlain() throws InterruptedException { server.answer(101, true); Countly.instance().init(config(server).enableRequestCompression().enableForcedHTTPPost()); awaitSettings(101); queueRequest("small", 300); - Received small = server.await(withMarker("small")); + GzipStandInRequest small = server.await(withMarker("small")); queueRequest("big", BIG); - Received big = server.await(withMarker("big")); + GzipStandInRequest big = server.await(withMarker("big")); Assert.assertEquals("POST", small.method); Assert.assertFalse(small.compressed); @@ -138,17 +125,17 @@ public void compressionOn_smallRequestStaysPlain() throws Exception { * A settings answer without the gzip key keeps every body plain. */ @Test - public void compressionOn_serverDoesNotSayGzip_requestsArePlain() throws Exception { + public void compressionOn_serverDoesNotSayGzip_requestsArePlain() throws InterruptedException { server.answer(101, false); Countly.instance().init(config(server).enableRequestCompression()); awaitSettings(101); queueRequest("big", BIG); - Received big = server.await(withMarker("big")); + GzipStandInRequest big = server.await(withMarker("big")); Assert.assertEquals("1", server.await(withMethod("sc")).params.get("gzip")); Assert.assertFalse(big.compressed); - Assert.assertFalse(provider().getRequestCompressionEnabled()); + Assert.assertFalse(provider().isRequestCompressionAllowed()); } /** @@ -156,15 +143,15 @@ public void compressionOn_serverDoesNotSayGzip_requestsArePlain() throws Excepti * stays on for the requests after it. */ @Test - public void compressionOn_bodyOverOneMegabyteStaysPlain_compressionStaysOn() throws Exception { + public void compressionOn_bodyOverOneMegabyteStaysPlain_compressionStaysOn() throws InterruptedException { server.answer(101, true); Countly.instance().init(config(server).enableRequestCompression()); awaitSettings(101); queueRequest("huge", Utils.REQUEST_COMPRESSION_MAX_BYTES); - Received huge = server.await(withMarker("huge")); + GzipStandInRequest huge = server.await(withMarker("huge")); queueRequest("big", BIG); - Received big = server.await(withMarker("big")); + GzipStandInRequest big = server.await(withMarker("big")); Assert.assertFalse(huge.compressed); Assert.assertTrue(huge.bodyBytes > Utils.REQUEST_COMPRESSION_MAX_BYTES); @@ -176,9 +163,9 @@ public void compressionOn_bodyOverOneMegabyteStaysPlain_compressionStaysOn() thr * requests after it. */ @Test - public void compressedRequestRefused_retryAndLaterRequestsArePlain() throws Exception { + public void compressedRequestRefused_retryAndLaterRequestsArePlain() throws InterruptedException { server.answer(101, true); - server.refuseCompressed = 1; + server.refuseCompressed.set(1); Countly.instance().init(config(server).enableRequestCompression()); awaitSettings(101); @@ -188,9 +175,9 @@ public void compressedRequestRefused_retryAndLaterRequestsArePlain() throws Exce Countly.instance().attemptToSendStoredRequests(); Assert.assertTrue(TestUtils.waitFor(15_000, () -> server.count(withMarker("refused")) == 2)); queueRequest("after", BIG); - Received after = server.await(withMarker("after")); + GzipStandInRequest after = server.await(withMarker("after")); - List refused = server.all(withMarker("refused")); + List refused = server.all(withMarker("refused")); Assert.assertEquals(Arrays.asList(true, false), compressedFlags(refused)); Assert.assertEquals(refused.get(0).params, refused.get(1).params); Assert.assertFalse(after.compressed); @@ -201,28 +188,28 @@ public void compressedRequestRefused_retryAndLaterRequestsArePlain() throws Exce * next one with "gzip": true turns it on again. */ @Test - public void compressedRequestRefused_nextGzipAnswerTurnsCompressionOnAgain() throws Exception { + public void compressedRequestRefused_nextGzipAnswerTurnsCompressionOnAgain() throws InterruptedException { server.answer(101, true); - server.refuseCompressed = 1; + server.refuseCompressed.set(1); Countly.instance().init(config(server).enableRequestCompression()); awaitSettings(101); queueRequest("refused", BIG); server.await(withMarker("refused")); - Assert.assertTrue(TestUtils.waitFor(10_000, () -> !provider().getRequestCompressionEnabled())); + Assert.assertTrue(TestUtils.waitFor(10_000, () -> !provider().isRequestCompressionAllowed())); server.answer(102, false); fetchSettings(); awaitSettings(102); - Assert.assertFalse(provider().getRequestCompressionEnabled()); + Assert.assertFalse(provider().isRequestCompressionAllowed()); server.answer(103, true); fetchSettings(); awaitSettings(103); - Assert.assertTrue(TestUtils.waitFor(10_000, () -> provider().getRequestCompressionEnabled())); + Assert.assertTrue(TestUtils.waitFor(10_000, () -> provider().isRequestCompressionAllowed())); awaitSendHandled(); Countly.instance().attemptToSendStoredRequests(); queueRequest("after_gzip_answer", BIG); - Received after = server.await(withMarker("after_gzip_answer")); + GzipStandInRequest after = server.await(withMarker("after_gzip_answer")); Assert.assertTrue(after.compressed); Assert.assertEquals(Arrays.asList(true, false), compressedFlags(server.all(withMarker("refused")))); @@ -232,7 +219,7 @@ public void compressedRequestRefused_nextGzipAnswerTurnsCompressionOnAgain() thr * A compressed remote config download answered with JSON leaves compression on. */ @Test - public void compressedImmediateRequestAnsweredWithJson_compressionStaysOn() throws Exception { + public void compressedImmediateRequestAnsweredWithJson_compressionStaysOn() throws InterruptedException { Assert.assertTrue(immediateRequestKeepsCompression(200, "{\"color\":\"red\"}")); } @@ -241,7 +228,7 @@ public void compressedImmediateRequestAnsweredWithJson_compressionStaysOn() thro * may send, was not accepted: compression stops. */ @Test - public void compressedImmediateRequestAnsweredWithPage_stopsCompression() throws Exception { + public void compressedImmediateRequestAnsweredWithPage_stopsCompression() throws InterruptedException { Assert.assertFalse(immediateRequestKeepsCompression(200, "Bad gateway")); } @@ -250,7 +237,7 @@ public void compressedImmediateRequestAnsweredWithPage_stopsCompression() throws * compression stops. */ @Test - public void compressedImmediateRequestAnsweredWithError_stopsCompression() throws Exception { + public void compressedImmediateRequestAnsweredWithError_stopsCompression() throws InterruptedException { Assert.assertFalse(immediateRequestKeepsCompression(400, "{\"result\":\"Invalid request body\"}")); } @@ -258,7 +245,7 @@ public void compressedImmediateRequestAnsweredWithError_stopsCompression() throw * A compressed remote config download answered with nothing was not accepted: compression stops. */ @Test - public void compressedImmediateRequestAnsweredWithNothing_stopsCompression() throws Exception { + public void compressedImmediateRequestAnsweredWithNothing_stopsCompression() throws InterruptedException { Assert.assertFalse(immediateRequestKeepsCompression(200, "")); } @@ -267,32 +254,32 @@ public void compressedImmediateRequestAnsweredWithNothing_stopsCompression() thr * goes compressed again. */ @Test - public void compressedRequestWithoutAnswer_compressionStaysOn() throws Exception { + public void compressedRequestWithoutAnswer_compressionStaysOn() throws InterruptedException { server.answer(101, true); - server.dropCompressed = true; + server.dropCompressed.set(true); Countly.instance().init(config(server).enableRequestCompression()); awaitSettings(101); queueRequest("dropped", BIG); server.await(withMarker("dropped")); awaitSendHandled(); - server.dropCompressed = false; + server.dropCompressed.set(false); Countly.instance().attemptToSendStoredRequests(); - Assert.assertTrue(TestUtils.waitFor(15_000, () -> server.count(withMarker("dropped").and(r -> r.answered)) == 1)); + Assert.assertTrue(TestUtils.waitFor(15_000, () -> server.count(withMarker("dropped").and(r -> r.answered.get())) == 1)); - List dropped = server.all(withMarker("dropped")); + List dropped = server.all(withMarker("dropped")); Assert.assertTrue(dropped.size() >= 2); - for (Received received : dropped) { + for (GzipStandInRequest received : dropped) { Assert.assertTrue(received.compressed); } - Assert.assertTrue(provider().getRequestCompressionEnabled()); + Assert.assertTrue(provider().isRequestCompressionAllowed()); } /** * The gzip key is read from the live answer only, never stored with the settings. */ @Test - public void gzipAnswer_isNotStored() throws Exception { + public void gzipAnswer_isNotStored() throws InterruptedException { server.answer(101, true); Countly.instance().init(config(server).enableRequestCompression()); awaitSettings(101); @@ -310,23 +297,23 @@ public void gzipAnswer_isNotStored() throws Exception { * After a server URL change the requests go plain, until the new server says it reads gzip. */ @Test - public void serverUrlChange_newServerGetsPlainRequestsUntilItSaysGzip() throws Exception { + public void serverUrlChange_newServerGetsPlainRequestsUntilItSaysGzip() throws IOException, InterruptedException { server.answer(101, true); - otherServer = new Server(); + otherServer = new GzipStandInServer(); otherServer.answer(102, true); CountDownLatch release = new CountDownLatch(1); - otherServer.holdSettings = release; + otherServer.holdSettings.set(release); Countly.instance().init(config(server).enableRequestCompression()); awaitSettings(101); Countly.instance().changeServerUrl(otherServer.url()); otherServer.await(withMethod("sc")); queueRequest("before_answer", BIG); - Received before = otherServer.await(withMarker("before_answer")); + GzipStandInRequest before = otherServer.await(withMarker("before_answer")); release.countDown(); awaitSettings(102); queueRequest("after_answer", BIG); - Received after = otherServer.await(withMarker("after_answer")); + GzipStandInRequest after = otherServer.await(withMarker("after_answer")); Assert.assertFalse(before.compressed); Assert.assertTrue(after.compressed); @@ -343,7 +330,7 @@ public void extractGzipFlag_onlyTrueCounts() { Assert.assertFalse(accepted.has("gzip")); for (String answer : new String[] { "{\"gzip\":false}", "{\"gzip\":\"true\"}", "{\"gzip\":1}", "{\"c\":{\"gzip\":true}}", "{}" }) { - Assert.assertFalse(answer, ModuleConfiguration.extractGzipFlag(new JSONObject(answer))); + Assert.assertFalse(answer, gzipFlagOf(answer)); } Assert.assertFalse(ModuleConfiguration.extractGzipFlag(null)); } @@ -361,7 +348,7 @@ public void gzipRequestBody_bounds() throws IOException { String body = "app_key=key&payload=" + repeat('a', size - 20); byte[] compressed = Utils.gzipRequestBody(body, log); Assert.assertTrue(compressed.length > 0 && compressed.length < body.length()); - Assert.assertEquals(body, new String(gunzip(compressed), StandardCharsets.UTF_8)); + Assert.assertEquals(body, gunzipText(compressed)); } } @@ -382,10 +369,10 @@ public void compressedRequestPath_markerIsTheOnlyQuery() { * Downloads remote config with 120 keys, a compressed POST, answered with the given status and * text, then tells whether a big request sent after it was compressed. */ - private boolean immediateRequestKeepsCompression(int remoteConfigCode, String remoteConfigAnswer) throws Exception { + private boolean immediateRequestKeepsCompression(int remoteConfigCode, String remoteConfigAnswer) throws InterruptedException { server.answer(101, true); - server.remoteConfigCode = remoteConfigCode; - server.remoteConfigAnswer = remoteConfigAnswer; + server.remoteConfigCode.set(remoteConfigCode); + server.remoteConfigAnswer.set(remoteConfigAnswer); Countly.instance().init(config(server, Config.Feature.RemoteConfig).enableRequestCompression()); awaitSettings(101); @@ -397,9 +384,9 @@ private boolean immediateRequestKeepsCompression(int remoteConfigCode, String re Countly.instance().remoteConfig().downloadSpecificKeys(keys, (result, error, fullValueUpdate, values) -> downloaded.countDown()); Assert.assertTrue(downloaded.await(15, TimeUnit.SECONDS)); queueRequest("after_download", BIG); - Received after = server.await(withMarker("after_download")); + GzipStandInRequest after = server.await(withMarker("after_download")); - Received download = server.await(withMethod("rc")); + GzipStandInRequest download = server.await(withMethod("rc")); Assert.assertTrue(download.compressed); Assert.assertEquals(120, new org.json.JSONArray(download.params.get("keys")).length()); return after.compressed; @@ -408,7 +395,7 @@ private boolean immediateRequestKeepsCompression(int remoteConfigCode, String re /** * A configuration that sends to the given server without request cooldowns or health check. */ - private static Config config(Server server, Config.Feature... features) { + private static Config config(GzipStandInServer server, Config.Feature... features) { File directory = TestUtils.getTestSDirectory(); TestUtils.checkSdkStorageRootDirectoryExist(directory); @@ -466,21 +453,21 @@ private static void awaitSendHandled() throws InterruptedException { /** * Matches the request that carries the given marker parameter. */ - private static Predicate withMarker(String marker) { + private static Predicate withMarker(String marker) { return r -> marker.equals(r.params.get("marker")); } /** * Matches an immediate request of the given method. */ - private static Predicate withMethod(String method) { + private static Predicate withMethod(String method) { return r -> method.equals(r.params.get("method")); } /** * Whether each request came compressed. */ - private static List compressedFlags(List requests) { + private static List compressedFlags(List requests) { return requests.stream().map(r -> r.compressed).collect(Collectors.toList()); } @@ -494,177 +481,17 @@ private static String repeat(char c, int length) { } /** - * Decodes gzip bytes. + * Reads the gzip flag of a settings answer given as text. */ - private static byte[] gunzip(byte[] compressed) throws IOException { - try (InputStream in = new GZIPInputStream(new ByteArrayInputStream(compressed))) { - return readAll(in); - } + private static boolean gzipFlagOf(String answer) { + return ModuleConfiguration.extractGzipFlag(new JSONObject(answer)); } /** - * Reads a stream to its end. + * Decodes gzip bytes into text. */ - private static byte[] readAll(InputStream stream) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; - int read; - while ((read = stream.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - return out.toByteArray(); - } - - /** - * One request as a server received it: the body decoded and its parameters read the way the - * Countly server reads them. - */ - private static final class Received { - final String method; - final String path; - final boolean compressed; - final String body; - final int bodyBytes; - final String contentType; - final String contentEncoding; - final Map params; - volatile boolean answered = false; - - /** - * Reads a request from its exchange. - */ - Received(HttpExchange exchange) throws IOException { - method = exchange.getRequestMethod(); - path = exchange.getRequestURI().getPath(); - String query = exchange.getRequestURI().getRawQuery(); - compressed = MARKER_QUERY.equals(query); - byte[] raw = readAll(exchange.getRequestBody()); - bodyBytes = raw.length; - body = new String(compressed ? gunzip(raw) : raw, StandardCharsets.UTF_8); - contentType = exchange.getRequestHeaders().getFirst("Content-Type"); - contentEncoding = exchange.getRequestHeaders().getFirst("Content-Encoding"); - params = TestUtils.parseQueryParams(compressed || !Utils.isNotEmpty(query) ? body : query); - params.replaceAll((key, value) -> Utils.urldecode(value)); - } - } - - /** - * A Countly server stand-in on loopback that records every request. It answers the SDK behavior - * settings with the answer set by {@link #answer(int, boolean)}, remote config with - * {@link #remoteConfigCode} and {@link #remoteConfigAnswer}, and anything else with success; it can refuse or drop compressed - * requests and hold its settings answer until a latch opens. - */ - private static final class Server { - final HttpServer http; - final List received = new CopyOnWriteArrayList<>(); - volatile String settingsAnswer = "{}"; - volatile String remoteConfigAnswer = "{}"; - volatile int remoteConfigCode = 200; - volatile int refuseCompressed = 0; - volatile boolean dropCompressed = false; - volatile CountDownLatch holdSettings = null; - - /** - * Starts the server on a free loopback port. - */ - Server() throws IOException { - http = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); - http.createContext("/", this::handle); - http.setExecutor(Executors.newCachedThreadPool()); - http.start(); - } - - /** - * Sets the settings answer: an {@code lkl} value that tells it apart, with "gzip": true when - * the server reads compressed bodies. - */ - void answer(int lkl, boolean gzip) { - JSONObject answer = new JSONObject().put("v", 1).put("t", 1_700_000_000_000L).put("c", new JSONObject().put("lkl", lkl)); - if (gzip) { - answer.put("gzip", true); - } - settingsAnswer = answer.toString(); - } - - /** - * Records a request and answers it. - */ - private void handle(HttpExchange exchange) throws IOException { - Received request = new Received(exchange); - received.add(request); - String method = request.params.get("method"); - int code = 200; - String response = "{\"result\":\"Success\"}"; - if ("sc".equals(method)) { - CountDownLatch hold = holdSettings; - if (hold != null) { - try { - hold.await(15, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - response = settingsAnswer; - } else if ("rc".equals(method)) { - code = remoteConfigCode; - response = remoteConfigAnswer; - } - if (request.compressed && dropCompressed) { - exchange.close(); - return; - } - if (request.compressed && refuseCompressed > 0) { - refuseCompressed--; - code = 400; - response = "{\"result\":\"Invalid request body\"}"; - } - byte[] out = response.getBytes(StandardCharsets.UTF_8); - exchange.sendResponseHeaders(code, out.length == 0 ? -1 : out.length); - try (OutputStream stream = exchange.getResponseBody()) { - stream.write(out); - } - request.answered = true; - } - - /** - * The URL the SDK reaches this server at. - */ - String url() { - return "http://127.0.0.1:" + http.getAddress().getPort(); - } - - /** - * How many received requests match. - */ - int count(Predicate filter) { - return (int) received.stream().filter(filter).count(); - } - - /** - * Every received request that matches, in arrival order. - */ - List all(Predicate filter) { - return new ArrayList<>(received.stream().filter(filter).collect(Collectors.toList())); - } - - /** - * Waits for the first matching request and returns it. - */ - Received await(Predicate filter) throws InterruptedException { - Assert.assertTrue("no matching request arrived at " + url() + ", it received " + received.size(), TestUtils.waitFor(15_000, () -> count(filter) > 0)); - return received.stream().filter(filter).findFirst().orElse(null); - } - - /** - * Opens a held settings answer and stops the server. - */ - void stop() { - CountDownLatch hold = holdSettings; - if (hold != null) { - hold.countDown(); - } - http.stop(0); - } + private static String gunzipText(byte[] compressed) throws IOException { + return new String(GzipStandInRequest.gunzip(compressed), StandardCharsets.UTF_8); } // endregion