Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
* Added "disableSdkBehaviorSettingsUpdates()" init config method for disabling SDK behavior settings requests.
* Improved request queue handling with a built-in backoff mechanism which is enabled by default. When the server takes 10 seconds or longer to answer while few requests are queued, the SDK pauses sending for 60 seconds.
* Added "disableBackoffMechanism()" init config method to disable the backoff behavior.
* Added "enableRequestCompression()" init config method. Request bodies between 1 KB and 1 MB are then sent gzip-compressed, once the server's SDK behavior settings answer says it reads them. It does not apply in backend mode, which does not use the SDK behavior settings.
* Added support for server requested SDK log gathering. When the server asks for it, the SDK uploads its own internal log lines, even if logging is disabled. While consent is required, a line is uploaded only while the consent of the feature it belongs to is given.
* Added support for the SDK connection test. When the server asks for it, the SDK checks that it can reach every endpoint it depends on and reports the results.
* Added "setMaxKeyLength(int)" init config method to limit the length of event keys, view names, segmentation keys and custom user property keys. Longer keys are truncated. There is no limit by default, and the key length limit of the SDK behavior settings overrides it.
Expand Down
4 changes: 4 additions & 0 deletions sdk-java/api/sdk-java.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ package ly.count.sdk.java {
method public ly.count.sdk.java.Config! enableParameterTamperingProtection(String!);
method public ly.count.sdk.java.Config! enableRemoteConfigAutomaticTriggers();
method public ly.count.sdk.java.Config! enableRemoteConfigValueCaching();
method public ly.count.sdk.java.Config! enableRequestCompression();
method @Deprecated public ly.count.sdk.java.Config! enableTestMode();
method @Deprecated public ly.count.sdk.java.Config! enableUsePOST();
method public ly.count.sdk.java.Config! enrollABOnRCDownload();
Expand Down Expand Up @@ -131,6 +132,7 @@ package ly.count.sdk.java {
field protected int networkRequestCooldown;
field protected Set<String!>! publicKeyPins;
field protected List<ly.count.sdk.java.internal.RCDownloadCallback!>! remoteConfigGlobalCallbacks;
field protected boolean requestCompressionEnabled;
field protected int requestQueueMaxSize;
field protected boolean requiresConsent;
field protected String! salt;
Expand Down Expand Up @@ -407,6 +409,7 @@ package ly.count.sdk.java.internal {
}

public interface ConfigurationProvider {
method public default void blockRequestCompression();
method public boolean getAutomaticCrashReportingEnabled();
method public boolean getAutomaticSessionTrackingEnabled();
method public boolean getAutomaticViewTrackingEnabled();
Expand Down Expand Up @@ -449,6 +452,7 @@ package ly.count.sdk.java.internal {
method public int getUserPropertyCacheLimit();
method public ly.count.sdk.java.internal.ConfigurationProvider.FilterList<Set<String!>!> getUserPropertyFilterList();
method public boolean getViewTrackingEnabled();
method public default boolean isRequestCompressionAllowed();
method public boolean isRequestQueueMaxSizeFromBehaviorSettings();
}

Expand Down
22 changes: 22 additions & 0 deletions sdk-java/src/main/java/ly/count/sdk/java/Config.java
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,12 @@
protected int networkReadTimeout = 30;

/**
* How long to wait between requests in milliseconds. Used to decrease CPU & I/O load on the device.

Check warning on line 107 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
*/
protected int networkRequestCooldown = 1000;

/**
* How long to wait between Device ID change & push token requests, in milliseconds.

Check warning on line 112 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'

Check warning on line 112 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
* Required by Countly server, don't change unless you know what you're doing!
*/
protected int networkImportantRequestCooldown = 5000;
Expand All @@ -126,7 +126,7 @@
* This ensures pinning continues to work after certificate prolongation.
* Certificates ({@link #certificatePins}) on the other hand have specific expiry date.
* In case you chose this way of pinning, you MUST ensure that ALL installs of your app
* have both certificates (old & new) until expiry date.

Check warning on line 129 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
*/
protected Set<String> publicKeyPins = null;

Expand All @@ -142,7 +142,7 @@
* This ensures pinning continues to work after certificate prolongation.
* Certificates on the other hand have specific expiry date.
* In case you chose this way of pinning, you MUST ensure that ALL installs of your app
* have both certificates (old & new) until expiry date.

Check warning on line 145 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
*/
protected Set<String> certificatePins = null;

Expand All @@ -168,7 +168,7 @@
/**
* {@link CrashProcessor}-implementing class which is instantiated when application
* crashes or crash is reported programmatically using {@link Session#addCrashReport(Throwable, boolean, String, Map, String...)}.
* Crash processor helps you to add custom data in event of a crash: custom crash segments & crash logs.

Check warning on line 171 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
*/
protected String crashProcessorClass = null;

Expand Down Expand Up @@ -272,6 +272,12 @@
*/
protected boolean sdkBehaviorSettingsRequestsDisabled = false;

/**
* Whether request bodies are sent gzip-compressed, once the server's SDK behavior settings
* answer says it reads them.
*/
protected boolean requestCompressionEnabled = false;

/**
* Whether the backoff mechanism may hold the request queue back after the server answered
* slowly. The SDK behavior settings can override it.
Expand Down Expand Up @@ -718,7 +724,7 @@
}

/**
* Disable update requests completely. Only begin & end requests will be sent + some special

Check warning on line 727 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
* cases if applicable like User Profile change or Push token updated.
*
* @return {@code this} instance for method chaining
Expand Down Expand Up @@ -816,7 +822,7 @@

/**
* How long to wait between requests in seconds.
* Used to decrease CPU & I/O load on the device in case of batch requests.

Check warning on line 825 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
*
* @param milliseconds cooldown period in seconds
* @return {@code this} instance for method chaining
Expand All @@ -834,7 +840,7 @@

/**
* Set read timeout in seconds for HTTP requests SDK sends to Countly server. Defaults to 30.
* Used to decrease CPU & I/O load on the device in case of batch requests.

Check warning on line 843 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
*
* @param milliseconds read timeout in milliseconds
* @return {@code this} instance for method chaining
Expand Down Expand Up @@ -862,7 +868,7 @@
* This ensures pinning continues to work after certificate prolongation.
* Certificates ({@link #certificatePins}) on the other hand have specific expiry date.
* In case you chose this way of pinning, you MUST ensure that ALL installs of your app
* have both certificates (old & new) until expiry date.

Check warning on line 871 in sdk-java/src/main/java/ly/count/sdk/java/Config.java

View workflow job for this annotation

GitHub Actions / Release checks (26.8.0)

invalid input: '&'
*
* NOTE: when {@link #serverURL} doesn't have {@code "https://"} public key pinning doesn't work
*
Expand Down Expand Up @@ -1588,6 +1594,22 @@
return this;
}

/**
* Compress request bodies between 1 KB and 1 MB with gzip before sending them. The SDK
* compresses only after the server's SDK behavior settings answer said it reads compressed
* bodies, so older servers keep getting plain requests. It needs the SDK behavior settings
* updates, which backend mode and {@link #disableSdkBehaviorSettingsUpdates()} turn off. A
* compressed request that the server, or something in front of it, answers without accepting
* it stops compression until the next SDK behavior settings answer says again that the server
* reads compressed bodies.
*
* @return {@code this} instance for method chaining
*/
public Config enableRequestCompression() {
this.requestCompressionEnabled = true;
return this;
}

/**
* Disable the backoff mechanism. While it is on, a request the server takes long to answer
* makes the SDK wait before it sends the next queued request, which gives a busy server room
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,25 @@ public interface ConfigurationProvider {
*/
@Nonnull Set<String> getJourneyTriggerViews();

/**
* Whether a request body may be sent gzip-compressed now: the developer turned request
* compression on, the current server said in its SDK behavior settings answer that it reads
* compressed bodies, and no compressed request was refused since that answer.
*
* @return {@code true} when a request body may be compressed
*/
default boolean isRequestCompressionAllowed() {
return false;
}

/**
* Stops request compression until the next SDK behavior settings answer says that the server
* reads compressed bodies, or the SDK starts again; called after the server, or something in
* front of it, answered a compressed request without accepting it.
*/
default void blockRequestCompression() {
}

/**
* The decision of the {@code lg} directive, which sits next to {@code c} in a live response.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.IOException;
import java.net.HttpURLConnection;
import java.util.concurrent.CompletableFuture;
import org.json.JSONException;
import org.json.JSONObject;

/**
Expand Down Expand Up @@ -93,8 +94,12 @@ private JSONObject doInBackground(String requestData, String customEndpoint, Tra
return new JSONObject(receivedBuffer);
} else {
L.d("[ImmediateRequestMaker] Encountered problem while making a immediate server request, :[" + receivedBuffer + "]");
cp.blockCompressionIfCompressed(connection);
return null;
}
} catch (JSONException | StringIndexOutOfBoundsException e) {
L.d("[ImmediateRequestMaker] Received an answer that is empty or not JSON while making a immediate server request, [" + e.getMessage() + "]");
cp.blockCompressionIfCompressed(connection);
} catch (Exception e) {
L.d("[ImmediateRequestMaker] Received exception while making a immediate server request " + e.getMessage());
} finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,15 @@ public boolean isSdkBehaviorSettingsRequestsDisabled() {
return sdkBehaviorSettingsRequestsDisabled;
}

/**
* Whether the developer turned request compression on, see {@link Config#enableRequestCompression()}.
*
* @return {@code true} after {@link Config#enableRequestCompression()}
*/
boolean isRequestCompressionEnabled() {
return requestCompressionEnabled;
}

/**
* Whether the SDK sends its health check, see {@link Config#disableHealthCheck()}.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import ly.count.sdk.java.Config;
Expand Down Expand Up @@ -38,6 +40,8 @@ public class ModuleConfiguration extends ModuleBase implements ConfigurationProv
static final String keyRConfig = "c";
static final String keyRLogGathering = "lg";
static final String keyRConnectionTest = "ct";
//top level key, a sibling of 'c', read from a live response and never stored
static final String keyRGzip = "gzip";

static final String keyRTracking = "tracking";
static final String keyRNetworking = "networking";
Expand Down Expand Up @@ -112,6 +116,10 @@ public class ModuleConfiguration extends ModuleBase implements ConfigurationProv

private boolean serverConfigRequestsDisabled = false;
private volatile boolean running = false;

private final AtomicBoolean requestCompressionEnabled = new AtomicBoolean(false);
final AtomicReference<String> gzipAcceptingServerUrl = new AtomicReference<>(null);
final AtomicBoolean requestCompressionBlocked = new AtomicBoolean(false);
/**
* How many times the device ID changed without merge, written under this module's lock, so a
* response requested before such a change can tell its gather was armed for the previous user.
Expand Down Expand Up @@ -230,6 +238,7 @@ public void init(InternalConfig config) {
super.init(config);
boolean backendMode = config.isBackendModeEnabled();
serverConfigRequestsDisabled = backendMode || config.isSdkBehaviorSettingsRequestsDisabled();
requestCompressionEnabled.set(config.isRequestCompressionEnabled());
L.d("[ModuleConfiguration] init, backend mode:[" + backendMode + "], server config requests disabled:[" + serverConfigRequestsDisabled + "]");

seedFromDeveloperConfig(config);
Expand Down Expand Up @@ -899,7 +908,11 @@ void fetchConfigFromServer(@Nonnull InternalConfig config) {
}

try {
String requestData = ModuleRequests.prepareRequiredParams(config).add("method", "sc").toString();
Params params = ModuleRequests.prepareRequiredParams(config).add("method", "sc");
if (requestCompressionEnabled.get()) {
params.add("gzip", "1");
}
String requestData = params.toString();
Transport transport = config.sdk.networking.getTransport();
final long fetchStartNs = System.nanoTime();
final int requestedAfterDeviceIdChanges = deviceIdChangesWithoutMerge;
Expand Down Expand Up @@ -946,6 +959,14 @@ private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JS
L.d("[ModuleConfiguration] onServerConfigResponse, retrieved configuration response: [" + response + "]");
long fetchLatencyMs = (System.nanoTime() - fetchStartNs) / 1_000_000L;
boolean connectionTestRequested = extractConnectionTestFlag(response);
boolean gzipAccepted = extractGzipFlag(response);
if (requestCompressionEnabled.get()) {
gzipAcceptingServerUrl.set(gzipAccepted ? requestedServerUrl : null);
if (gzipAccepted && requestCompressionBlocked.compareAndSet(true, false)) {
L.i("[ModuleConfiguration] onServerConfigResponse, a behavior settings answer says that the server reads compressed bodies, requests are compressed again");
}
L.d("[ModuleConfiguration] onServerConfigResponse, the server " + (gzipAccepted ? "reads" : "does not say it reads") + " compressed request bodies");
}

boolean changed;
synchronized (this) {
Expand All @@ -967,6 +988,21 @@ private void onServerConfigResponse(@Nonnull InternalConfig config, @Nullable JS
}
}

/**
* Reads and removes the {@code gzip} flag of a live response, so it is never stored: only an
* answer that carries {@code true} says that the server reads compressed request bodies.
*
* @param serverConfigResponse the live response, changed in place
* @return {@code true} only when the response had {@code "gzip": true}
*/
static boolean extractGzipFlag(@Nullable JSONObject serverConfigResponse) {
if (serverConfigResponse == null || !serverConfigResponse.has(keyRGzip)) {
return false;
}

return Boolean.TRUE.equals(serverConfigResponse.remove(keyRGzip));
}

/**
* Reads and removes the {@code ct} flag of a live response, so it is never stored and can never
* come back from storage. Booleans and numbers count by value; a string counts unless it is
Expand Down Expand Up @@ -1060,6 +1096,21 @@ private synchronized void stopServerConfigUpdateTimer() {
}

/** {@inheritDoc} */
@Override
public boolean isRequestCompressionAllowed() {
String acceptingUrl = gzipAcceptingServerUrl.get();
InternalConfig config = internalConfig;
return requestCompressionEnabled.get() && !requestCompressionBlocked.get() && acceptingUrl != null && config != null
&& acceptingUrl.equals(config.getServerURL().toString());
}

@Override
public void blockRequestCompression() {
if (requestCompressionBlocked.compareAndSet(false, true)) {
L.i("[ModuleConfiguration] blockRequestCompression, a compressed request was refused, requests are sent uncompressed until a behavior settings answer says that the server reads compressed bodies, or the SDK starts again");
}
}

@Override
public boolean getNetworkingEnabled() {
return currentVNetworking;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ public class Transport implements X509TrustManager {
private Log L = null;
private static final String PARAMETER_TAMPERING_DIGEST = "SHA-256";
private static final String CHECKSUM = "checksum256";
static final String REQUEST_COMPRESSION_MARKER = "content_encoding=gzip";
private InternalConfig config;

private SSLContext sslContext; // ssl context to use if pinning is enabled
Expand Down Expand Up @@ -150,6 +151,22 @@ HttpURLConnection connection(final Request request) throws IOException {
request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L));
}

byte[] compressedBody = new byte[0];
if (!usingGET && maybePictureData == null) {
if (config.getParameterTamperingProtectionSalt() != null) {
request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L));
}
String markedPath = compressedRequestPath(path);
if (markedPath != null && config.getConfigurationProvider().isRequestCompressionAllowed()) {
String body = request.params.toString();
compressedBody = Utils.gzipRequestBody(body, L);
if (compressedBody.length > 0) {
path = markedPath;
L.d("[Transport] connection, the request body is sent gzip-compressed, [" + body.length() + "] -> [" + compressedBody.length + "] bytes");
}
}
}

HttpURLConnection connection = openConnection(path, request.params.toString(), usingGET);
connection.setConnectTimeout(1000 * config.getNetworkConnectionTimeout());
connection.setReadTimeout(1000 * config.getNetworkReadTimeout());
Expand Down Expand Up @@ -191,17 +208,19 @@ HttpURLConnection connection(final Request request) throws IOException {
writer.append("--").append(boundary).append("--").append(Utils.CRLF).flush();
} else {
//picture data is "null". If it was sent, we send "null" to server to clear the image there
//we send a normal request in HTTP POST
if (config.getParameterTamperingProtectionSalt() != null) {
request.params.add(CHECKSUM, Utils.digestHex(PARAMETER_TAMPERING_DIGEST, request.params + config.getParameterTamperingProtectionSalt(), L));
}
//we send a normal request in HTTP POST, its checksum was added above
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");

output = connection.getOutputStream();
writer = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true);
if (compressedBody.length > 0) {
output.write(compressedBody);
output.flush();
} else {
writer = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8), true);

writer.write(request.params.toString());
writer.flush();
writer.write(request.params.toString());
writer.flush();
}
}
} finally {
if (writer != null) {
Expand All @@ -222,6 +241,44 @@ HttpURLConnection connection(final Request request) throws IOException {
return connection;
}

/**
* The URL a compressed request goes to: the request URL with the compression marker as its only
* query. Endpoints end with an empty query, as in {@code /i?}, which the marker fills.
*
* @param url the URL the request goes to
* @return the URL with the marker, or {@code null} when the URL already has a query, where the
* marker could not be the only one
*/
static String compressedRequestPath(@Nonnull String url) {
int query = url.indexOf('?');
if (query == -1) {
return url + "?" + REQUEST_COMPRESSION_MARKER;
}
return query == url.length() - 1 ? url + REQUEST_COMPRESSION_MARKER : null;
}

/**
* Tells whether a connection made by {@link #connection(Request)} sends its body gzip-compressed.
*
* @param connection the connection
* @return {@code true} when its URL carries the compression marker as its query
*/
static boolean isCompressedRequest(@Nullable HttpURLConnection connection) {
return connection != null && REQUEST_COMPRESSION_MARKER.equals(connection.getURL().getQuery());
}

/**
* Stops request compression when the connection sent a compressed body; called after its answer
* was not accepted.
*
* @param connection the connection whose answer was not accepted
*/
void blockCompressionIfCompressed(@Nullable HttpURLConnection connection) {
if (isCompressedRequest(connection)) {
config.getConfigurationProvider().blockRequestCompression();
}
}

/**
* Opens the GET of one connection test probe: a bare request, built as
* {@link #openBareGetConnection(String, int)} builds it, that verifies the server against the
Expand Down Expand Up @@ -444,7 +501,11 @@ public Boolean send() {

SDKCore.instance.onRequestCompleted(request, response, code, requestOwner);

return processResponse(code, response, request.storageId());
Boolean accepted = processResponse(code, response, request.storageId());
if (!Boolean.TRUE.equals(accepted)) {
blockCompressionIfCompressed(connection);
}
return accepted;
} catch (IOException e) {
L.d("[network] Error while sending request " + request + " " + e);
notifyOwnerOfAttemptWithoutResponse(request, requestOwner);
Expand Down
Loading
Loading