Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
- Added the method `change_server_url` that changes the server URL after init
- Added the method `change_app_key` that changes the app key after init
- Added the queue methods `flush_queues`, `replace_all_app_keys_in_queue_with_current_app_key` and `remove_different_app_keys_from_queue`
- Added the `enable_request_compression` init option. Request bodies between 1 KB and 1 MB are then sent gzip-compressed, once the server's SDK behavior settings answer says it reads them

- Mitigated an issue where the health check counted every failed request as an SDK error
- Mitigated an issue where every call without consent was logged as an error
Expand Down
17 changes: 17 additions & 0 deletions lib/countly-behavior.js
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,22 @@ function parseAnswer(responseText, warn) {
return normalize(answer.c, warn);
}

/**
* Tells whether a settings answer says that the server reads gzip-compressed request bodies: a JSON object with
* "gzip": true next to "c". The key is read from each live answer and never stored.
* @param {String} responseText - the answer body
* @returns {Boolean} true only for an answer with "gzip": true
*/
function readsCompressedBodies(responseText) {
try {
var answer = JSON.parse(responseText);
return isPlainObject(answer) && answer.gzip === true;
}
catch (ex) {
return false;
}
}

/**
* Reads the settings a developer provides with the "behavior_settings" init option, in the same format the
* server sends: an object with a "c" object
Expand Down Expand Up @@ -215,6 +231,7 @@ module.exports = {
MAX_REFRESH_HOURS,
normalize,
parseAnswer,
readsCompressedBodies,
parseProvided,
parseStored,
valueOf,
Expand Down
205 changes: 154 additions & 51 deletions lib/countly.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ var http = require("http");
var https = require("https");
var fs = require("fs");
var path = require("path");
var zlib = require("zlib");
var cluster = require("cluster");
var cc = require("./countly-common");
var behavior = require("./countly-behavior");
Expand All @@ -45,6 +46,9 @@ Countly.Bulk = Bulk;
var platform;
var apiPath = "/i";
var readPath = "/o/sdk";
var REQUEST_COMPRESSION_MIN_BYTES = 1024;
var REQUEST_COMPRESSION_MAX_BYTES = 1048576;
var REQUEST_COMPRESSION_MARKER = "content_encoding=gzip";
var beatInterval = 500;
var queueSize = 1000;
var requestQueue = [];
Expand Down Expand Up @@ -101,6 +105,9 @@ Countly.Bulk = Bulk;
var health = HealthCounters.create(healthAreaAllowed);
var healthSavedAt = 0;
var backoffUntil = 0;
var requestCompressionEnabled = false;
var gzipAcceptingUrl = null;
var requestCompressionBlocked = false;
/**
* Array with list of available features that you can require consent for
*/
Expand Down Expand Up @@ -174,6 +181,7 @@ Countly.Bulk = Bulk;
* @param {boolean} [conf.disable_sdk_behavior_settings_updates=false] - do not fetch the SDK behavior settings from the server; stored and provided settings still apply
* @param {boolean} [conf.disable_health_check=false] - do not send the health check at start
* @param {boolean} [conf.disable_backoff_mechanism=false] - never pause sending when the server answers slowly, whatever the server settings say
* @param {boolean} [conf.enable_request_compression=false] - send request bodies between 1 KB and 1 MB gzip-compressed, once the server's SDK behavior settings answer says it reads them
* @example
* Countly.init({
* app_key: "{YOUR-APP-KEY}",
Expand Down Expand Up @@ -238,6 +246,7 @@ Countly.Bulk = Bulk;
settingsUpdatesDisabled = !!(conf.disable_sdk_behavior_settings_updates || Countly.disable_sdk_behavior_settings_updates);
healthCheckDisabled = !!(conf.disable_health_check || Countly.disable_health_check);
backoffDisabled = !!(conf.disable_backoff_mechanism || Countly.disable_backoff_mechanism);
requestCompressionEnabled = conf.enable_request_compression === true || Countly.enable_request_compression === true;
developerSettings = {
rqs: queueSize,
eqs: maxEventBatch,
Expand Down Expand Up @@ -304,6 +313,7 @@ Countly.Bulk = Bulk;
cc.log(cc.logLevelEnums.DEBUG, `init, IP address: [${Countly.ip_address}].`);
}
cc.log(cc.logLevelEnums.DEBUG, `init, Force POST requests: [${Countly.force_post}].`);
cc.log(cc.logLevelEnums.DEBUG, `init, Request compression: [${requestCompressionEnabled}].`);
cc.log(cc.logLevelEnums.DEBUG, `init, Salt is configured: [${!!Countly.salt}].`);
cc.log(cc.logLevelEnums.DEBUG, `init, Storage path: [${CountlyStorage.getStoragePath()}].`);
cc.log(cc.logLevelEnums.DEBUG, `init, Require consent: [${Countly.require_consent}].`);
Expand Down Expand Up @@ -469,6 +479,9 @@ Countly.Bulk = Bulk;
health = HealthCounters.create(healthAreaAllowed);
healthSavedAt = 0;
backoffUntil = 0;
requestCompressionEnabled = false;
gzipAcceptingUrl = null;
requestCompressionBlocked = false;
requestTimeout = 30;
standaloneWorker = false;
if (uncaughtExceptionHandler) {
Expand Down Expand Up @@ -513,6 +526,7 @@ Countly.Bulk = Bulk;
Countly.disable_sdk_behavior_settings_updates = undefined;
Countly.disable_health_check = undefined;
Countly.disable_backoff_mechanism = undefined;
Countly.enable_request_compression = undefined;
CountlyStorage.resetStorage();
};

Expand Down Expand Up @@ -2459,7 +2473,9 @@ Countly.Bulk = Bulk;
* Fetches the SDK behavior settings from the server and puts them into effect. The request is sent
* directly, also while networking is turned off, so that turning networking back on reaches the SDK. A
* failed request or an unusable answer keeps the current settings. The next fetch follows scui hours later.
* An answer that arrives after the app key or the server URL changed is not used.
* An answer that arrives after the app key or the server URL changed is not used. With request compression on, the
* request asks whether the server reads compressed bodies, and the "gzip" key of each answer turns compression on
* or off for that server.
* @param {Function=} done - called when the fetch finished or was skipped
*/
function fetchBehaviorSettings(done) {
Expand All @@ -2478,7 +2494,11 @@ Countly.Bulk = Bulk;
var generation = sdkGeneration;
var appKey = Countly.app_key;
var { url } = Countly;
makeRequest(url, readPath, directRequestParams({ method: "sc" }), (err, params, responseText) => {
var request = { method: "sc" };
if (requestCompressionEnabled) {
request.gzip = 1;
}
makeRequest(url, readPath, directRequestParams(request), (err, params, responseText) => {
if (generation !== sdkGeneration) {
return;
}
Expand All @@ -2487,6 +2507,15 @@ Countly.Bulk = Bulk;
finish();
return;
}
if (!err && requestCompressionEnabled) {
var gzipAccepted = behavior.readsCompressedBodies(responseText);
gzipAcceptingUrl = gzipAccepted ? url : null;
if (gzipAccepted && requestCompressionBlocked) {
requestCompressionBlocked = false;
cc.log(cc.logLevelEnums.INFO, "fetchBehaviorSettings, A behavior settings answer says that the server reads compressed bodies, requests are compressed again.");
}
cc.log(cc.logLevelEnums.DEBUG, `fetchBehaviorSettings, The server ${gzipAccepted ? "reads" : "does not say it reads"} compressed request bodies.`);
}
var fetched = err ? null : behavior.parseAnswer(responseText, warnAboutSettings);
if (fetched) {
serverSettings = fetched;
Expand Down Expand Up @@ -2993,6 +3022,7 @@ Countly.Bulk = Bulk;
*/
function makeRequest(url, api, params, callback, info, isBroad) {
var finished = false;
var generation = sdkGeneration;
/**
* Reports the outcome of the request to the callback, once
* @param {Boolean} failed - true if the request failed
Expand Down Expand Up @@ -3108,66 +3138,139 @@ Countly.Bulk = Bulk;
options.path = api;
options.headers["Content-Type"] = "application/x-www-form-urlencoded";
options.headers["Content-Length"] = Buffer.byteLength(data);
if (compressionAllowed(url)) {
compressRequestBody(data, (compressed) => send(options, method, data, compressed));
return;
}
}
}
send(options, method, data, null);
}
catch (e) {
// fallback
cc.log(cc.logLevelEnums.ERROR, "makeRequest, Failed the HTTP request. Error:", e);
finish(true);
}

if (typeof Countly.http_options === "function") {
Countly.http_options(options);
}
var protocol = https;
if (url.indexOf("http:") === 0) {
protocol = http;
}
var req = protocol.request(options, (res) => {
var str = "";
res.setEncoding("utf8");
res.on("data", (chunk) => {
str += chunk;
/**
* Sends the prepared request. A gzip-compressed body goes to the endpoint with the compression marker as the URL
* query, and an answer that does not accept it stops request compression
* @param {Object} requestOptions - request options
* @param {String} requestMethod - "GET" or "POST"
* @param {String|Buffer} body - the prepared body, sent when requestMethod is "POST"
* @param {Buffer=} compressed - the gzip-compressed body to send instead of body, or null
*/
function send(requestOptions, requestMethod, body, compressed) {
try {
if (compressed) {
requestOptions.path = `${api}?${REQUEST_COMPRESSION_MARKER}`;
requestOptions.headers["Content-Length"] = compressed.length;
}
if (typeof Countly.http_options === "function") {
Countly.http_options(requestOptions);
}
var protocol = https;
if (url.indexOf("http:") === 0) {
protocol = http;
}
var req = protocol.request(requestOptions, (res) => {
var str = "";
res.setEncoding("utf8");
res.on("data", (chunk) => {
str += chunk;
});
res.on("end", () => {
cc.log(cc.logLevelEnums.DEBUG, `makeRequest, Response status code: [${res.statusCode}], response: [${str}]`);
// response validation function will be selected to also accept JSON arrays if isBroad is true
var isResponseValidated;
if (isBroad) {
// JSON array/object both can pass
isResponseValidated = cc.isResponseValidBroad(res.statusCode, str);
}
else {
// only JSON object with result can pass
isResponseValidated = cc.isResponseValid(res.statusCode, str);
}
if (compressed && !isResponseValidated && generation === sdkGeneration) {
blockRequestCompression();
}
finish(!isResponseValidated, str, res.statusCode);
});
res.on("error", (err) => {
cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed while receiving the response. Error:", err);
finish(true);
});
res.on("close", () => {
if (!res.complete) {
cc.log(cc.logLevelEnums.DEBUG, "makeRequest, The connection closed before the whole response arrived.");
finish(true);
}
});
});
res.on("end", () => {
cc.log(cc.logLevelEnums.DEBUG, `makeRequest, Response status code: [${res.statusCode}], response: [${str}]`);
// response validation function will be selected to also accept JSON arrays if isBroad is true
var isResponseValidated;
if (isBroad) {
// JSON array/object both can pass
isResponseValidated = cc.isResponseValidBroad(res.statusCode, str);
}
else {
// only JSON object with result can pass
isResponseValidated = cc.isResponseValid(res.statusCode, str);
}
finish(!isResponseValidated, str, res.statusCode);
req.setTimeout(requestTimeout * 1000, () => {
cc.log(cc.logLevelEnums.DEBUG, `makeRequest, The server did not answer within [${requestTimeout}] seconds.`);
req.destroy(new Error("request timeout"));
});
res.on("error", (err) => {
cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed while receiving the response. Error:", err);
req.on("error", (err) => {
cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed. Error:", err);
finish(true);
});
res.on("close", () => {
if (!res.complete) {
cc.log(cc.logLevelEnums.DEBUG, "makeRequest, The connection closed before the whole response arrived.");
finish(true);
}
});
});
req.setTimeout(requestTimeout * 1000, () => {
cc.log(cc.logLevelEnums.DEBUG, `makeRequest, The server did not answer within [${requestTimeout}] seconds.`);
req.destroy(new Error("request timeout"));
});
req.on("error", (err) => {
cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed. Error:", err);
if (requestMethod === "POST") {
// write data to request body
req.write(compressed || body);
}

req.end();
}
catch (e) {
cc.log(cc.logLevelEnums.ERROR, "makeRequest, Failed the HTTP request. Error:", e);
finish(true);
});
if (method === "POST") {
// write data to request body
req.write(data);
}
}
}

/**
* Tells whether a request body may be sent gzip-compressed now: request compression is on, the server the request
* goes to said in its SDK behavior settings answer that it reads compressed bodies, and no compressed request was
* refused since that answer
* @param {String} url - server URL the request goes to
* @returns {Boolean} true when the body may be compressed
*/
function compressionAllowed(url) {
return requestCompressionEnabled && !requestCompressionBlocked && gzipAcceptingUrl !== null && gzipAcceptingUrl === url;
}

req.end();
/**
* Compresses a request body with gzip, on the thread pool instead of the main thread
* @param {String} body - url encoded request body, checksum included
* @param {Function} done - called with the gzip bytes, or with null when the body is smaller than 1 KB or larger than
* 1 MB, compressing fails, or the result is not smaller than the body
*/
function compressRequestBody(body, done) {
var plain = Buffer.from(body, "utf8");
if (plain.length < REQUEST_COMPRESSION_MIN_BYTES || plain.length > REQUEST_COMPRESSION_MAX_BYTES) {
done(null);
return;
}
catch (e) {
// fallback
cc.log(cc.logLevelEnums.ERROR, "makeRequest, Failed the HTTP request. Error:", e);
finish(true);
zlib.gzip(plain, (err, compressed) => {
if (err || compressed.length >= plain.length) {
done(null);
return;
}
cc.log(cc.logLevelEnums.DEBUG, `makeRequest, The request body is sent gzip-compressed, [${plain.length}] -> [${compressed.length}] bytes.`);
done(compressed);
});
}

/**
* Stops request compression after the server, or something in front of it, answered a compressed request without
* accepting it, until the next SDK behavior settings answer says that the server reads compressed bodies, or the SDK
* starts again
*/
function blockRequestCompression() {
if (!requestCompressionBlocked) {
requestCompressionBlocked = true;
cc.log(cc.logLevelEnums.INFO, "makeRequest, A compressed request was refused, requests are sent uncompressed until a behavior settings answer says that the server reads compressed bodies, or the SDK starts again.");
}
}

Expand Down
Loading
Loading