diff --git a/CHANGELOG.md b/CHANGELOG.md index ad9ddf5..e40f835 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Added the method `change_server_url` that changes the server URL after init - Added the method `change_app_key` that changes the app key after init - Added the queue methods `flush_queues`, `replace_all_app_keys_in_queue_with_current_app_key` and `remove_different_app_keys_from_queue` +- Added the `enable_request_compression` init option. Request bodies between 1 KB and 1 MB are then sent gzip-compressed, once the server's SDK behavior settings answer says it reads them - Mitigated an issue where the health check counted every failed request as an SDK error - Mitigated an issue where every call without consent was logged as an error diff --git a/lib/countly-behavior.js b/lib/countly-behavior.js index 5cb5c3e..2c3eff1 100644 --- a/lib/countly-behavior.js +++ b/lib/countly-behavior.js @@ -139,6 +139,22 @@ function parseAnswer(responseText, warn) { return normalize(answer.c, warn); } +/** + * Tells whether a settings answer says that the server reads gzip-compressed request bodies: a JSON object with + * "gzip": true next to "c". The key is read from each live answer and never stored. + * @param {String} responseText - the answer body + * @returns {Boolean} true only for an answer with "gzip": true + */ +function readsCompressedBodies(responseText) { + try { + var answer = JSON.parse(responseText); + return isPlainObject(answer) && answer.gzip === true; + } + catch (ex) { + return false; + } +} + /** * Reads the settings a developer provides with the "behavior_settings" init option, in the same format the * server sends: an object with a "c" object @@ -215,6 +231,7 @@ module.exports = { MAX_REFRESH_HOURS, normalize, parseAnswer, + readsCompressedBodies, parseProvided, parseStored, valueOf, diff --git a/lib/countly.js b/lib/countly.js index 854dabb..bdfab74 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -25,6 +25,7 @@ var http = require("http"); var https = require("https"); var fs = require("fs"); var path = require("path"); +var zlib = require("zlib"); var cluster = require("cluster"); var cc = require("./countly-common"); var behavior = require("./countly-behavior"); @@ -45,6 +46,9 @@ Countly.Bulk = Bulk; var platform; var apiPath = "/i"; var readPath = "/o/sdk"; + var REQUEST_COMPRESSION_MIN_BYTES = 1024; + var REQUEST_COMPRESSION_MAX_BYTES = 1048576; + var REQUEST_COMPRESSION_MARKER = "content_encoding=gzip"; var beatInterval = 500; var queueSize = 1000; var requestQueue = []; @@ -101,6 +105,9 @@ Countly.Bulk = Bulk; var health = HealthCounters.create(healthAreaAllowed); var healthSavedAt = 0; var backoffUntil = 0; + var requestCompressionEnabled = false; + var gzipAcceptingUrl = null; + var requestCompressionBlocked = false; /** * Array with list of available features that you can require consent for */ @@ -174,6 +181,7 @@ Countly.Bulk = Bulk; * @param {boolean} [conf.disable_sdk_behavior_settings_updates=false] - do not fetch the SDK behavior settings from the server; stored and provided settings still apply * @param {boolean} [conf.disable_health_check=false] - do not send the health check at start * @param {boolean} [conf.disable_backoff_mechanism=false] - never pause sending when the server answers slowly, whatever the server settings say + * @param {boolean} [conf.enable_request_compression=false] - send request bodies between 1 KB and 1 MB gzip-compressed, once the server's SDK behavior settings answer says it reads them * @example * Countly.init({ * app_key: "{YOUR-APP-KEY}", @@ -238,6 +246,7 @@ Countly.Bulk = Bulk; settingsUpdatesDisabled = !!(conf.disable_sdk_behavior_settings_updates || Countly.disable_sdk_behavior_settings_updates); healthCheckDisabled = !!(conf.disable_health_check || Countly.disable_health_check); backoffDisabled = !!(conf.disable_backoff_mechanism || Countly.disable_backoff_mechanism); + requestCompressionEnabled = conf.enable_request_compression === true || Countly.enable_request_compression === true; developerSettings = { rqs: queueSize, eqs: maxEventBatch, @@ -304,6 +313,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.DEBUG, `init, IP address: [${Countly.ip_address}].`); } cc.log(cc.logLevelEnums.DEBUG, `init, Force POST requests: [${Countly.force_post}].`); + cc.log(cc.logLevelEnums.DEBUG, `init, Request compression: [${requestCompressionEnabled}].`); cc.log(cc.logLevelEnums.DEBUG, `init, Salt is configured: [${!!Countly.salt}].`); cc.log(cc.logLevelEnums.DEBUG, `init, Storage path: [${CountlyStorage.getStoragePath()}].`); cc.log(cc.logLevelEnums.DEBUG, `init, Require consent: [${Countly.require_consent}].`); @@ -469,6 +479,9 @@ Countly.Bulk = Bulk; health = HealthCounters.create(healthAreaAllowed); healthSavedAt = 0; backoffUntil = 0; + requestCompressionEnabled = false; + gzipAcceptingUrl = null; + requestCompressionBlocked = false; requestTimeout = 30; standaloneWorker = false; if (uncaughtExceptionHandler) { @@ -513,6 +526,7 @@ Countly.Bulk = Bulk; Countly.disable_sdk_behavior_settings_updates = undefined; Countly.disable_health_check = undefined; Countly.disable_backoff_mechanism = undefined; + Countly.enable_request_compression = undefined; CountlyStorage.resetStorage(); }; @@ -2459,7 +2473,9 @@ Countly.Bulk = Bulk; * Fetches the SDK behavior settings from the server and puts them into effect. The request is sent * directly, also while networking is turned off, so that turning networking back on reaches the SDK. A * failed request or an unusable answer keeps the current settings. The next fetch follows scui hours later. - * An answer that arrives after the app key or the server URL changed is not used. + * An answer that arrives after the app key or the server URL changed is not used. With request compression on, the + * request asks whether the server reads compressed bodies, and the "gzip" key of each answer turns compression on + * or off for that server. * @param {Function=} done - called when the fetch finished or was skipped */ function fetchBehaviorSettings(done) { @@ -2478,7 +2494,11 @@ Countly.Bulk = Bulk; var generation = sdkGeneration; var appKey = Countly.app_key; var { url } = Countly; - makeRequest(url, readPath, directRequestParams({ method: "sc" }), (err, params, responseText) => { + var request = { method: "sc" }; + if (requestCompressionEnabled) { + request.gzip = 1; + } + makeRequest(url, readPath, directRequestParams(request), (err, params, responseText) => { if (generation !== sdkGeneration) { return; } @@ -2487,6 +2507,15 @@ Countly.Bulk = Bulk; finish(); return; } + if (!err && requestCompressionEnabled) { + var gzipAccepted = behavior.readsCompressedBodies(responseText); + gzipAcceptingUrl = gzipAccepted ? url : null; + if (gzipAccepted && requestCompressionBlocked) { + requestCompressionBlocked = false; + cc.log(cc.logLevelEnums.INFO, "fetchBehaviorSettings, A behavior settings answer says that the server reads compressed bodies, requests are compressed again."); + } + cc.log(cc.logLevelEnums.DEBUG, `fetchBehaviorSettings, The server ${gzipAccepted ? "reads" : "does not say it reads"} compressed request bodies.`); + } var fetched = err ? null : behavior.parseAnswer(responseText, warnAboutSettings); if (fetched) { serverSettings = fetched; @@ -2993,6 +3022,7 @@ Countly.Bulk = Bulk; */ function makeRequest(url, api, params, callback, info, isBroad) { var finished = false; + var generation = sdkGeneration; /** * Reports the outcome of the request to the callback, once * @param {Boolean} failed - true if the request failed @@ -3108,66 +3138,139 @@ Countly.Bulk = Bulk; options.path = api; options.headers["Content-Type"] = "application/x-www-form-urlencoded"; options.headers["Content-Length"] = Buffer.byteLength(data); + if (compressionAllowed(url)) { + compressRequestBody(data, (compressed) => send(options, method, data, compressed)); + return; + } } } + send(options, method, data, null); + } + catch (e) { + // fallback + cc.log(cc.logLevelEnums.ERROR, "makeRequest, Failed the HTTP request. Error:", e); + finish(true); + } - if (typeof Countly.http_options === "function") { - Countly.http_options(options); - } - var protocol = https; - if (url.indexOf("http:") === 0) { - protocol = http; - } - var req = protocol.request(options, (res) => { - var str = ""; - res.setEncoding("utf8"); - res.on("data", (chunk) => { - str += chunk; + /** + * Sends the prepared request. A gzip-compressed body goes to the endpoint with the compression marker as the URL + * query, and an answer that does not accept it stops request compression + * @param {Object} requestOptions - request options + * @param {String} requestMethod - "GET" or "POST" + * @param {String|Buffer} body - the prepared body, sent when requestMethod is "POST" + * @param {Buffer=} compressed - the gzip-compressed body to send instead of body, or null + */ + function send(requestOptions, requestMethod, body, compressed) { + try { + if (compressed) { + requestOptions.path = `${api}?${REQUEST_COMPRESSION_MARKER}`; + requestOptions.headers["Content-Length"] = compressed.length; + } + if (typeof Countly.http_options === "function") { + Countly.http_options(requestOptions); + } + var protocol = https; + if (url.indexOf("http:") === 0) { + protocol = http; + } + var req = protocol.request(requestOptions, (res) => { + var str = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { + str += chunk; + }); + res.on("end", () => { + cc.log(cc.logLevelEnums.DEBUG, `makeRequest, Response status code: [${res.statusCode}], response: [${str}]`); + // response validation function will be selected to also accept JSON arrays if isBroad is true + var isResponseValidated; + if (isBroad) { + // JSON array/object both can pass + isResponseValidated = cc.isResponseValidBroad(res.statusCode, str); + } + else { + // only JSON object with result can pass + isResponseValidated = cc.isResponseValid(res.statusCode, str); + } + if (compressed && !isResponseValidated && generation === sdkGeneration) { + blockRequestCompression(); + } + finish(!isResponseValidated, str, res.statusCode); + }); + res.on("error", (err) => { + cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed while receiving the response. Error:", err); + finish(true); + }); + res.on("close", () => { + if (!res.complete) { + cc.log(cc.logLevelEnums.DEBUG, "makeRequest, The connection closed before the whole response arrived."); + finish(true); + } + }); }); - res.on("end", () => { - cc.log(cc.logLevelEnums.DEBUG, `makeRequest, Response status code: [${res.statusCode}], response: [${str}]`); - // response validation function will be selected to also accept JSON arrays if isBroad is true - var isResponseValidated; - if (isBroad) { - // JSON array/object both can pass - isResponseValidated = cc.isResponseValidBroad(res.statusCode, str); - } - else { - // only JSON object with result can pass - isResponseValidated = cc.isResponseValid(res.statusCode, str); - } - finish(!isResponseValidated, str, res.statusCode); + req.setTimeout(requestTimeout * 1000, () => { + cc.log(cc.logLevelEnums.DEBUG, `makeRequest, The server did not answer within [${requestTimeout}] seconds.`); + req.destroy(new Error("request timeout")); }); - res.on("error", (err) => { - cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed while receiving the response. Error:", err); + req.on("error", (err) => { + cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed. Error:", err); finish(true); }); - res.on("close", () => { - if (!res.complete) { - cc.log(cc.logLevelEnums.DEBUG, "makeRequest, The connection closed before the whole response arrived."); - finish(true); - } - }); - }); - req.setTimeout(requestTimeout * 1000, () => { - cc.log(cc.logLevelEnums.DEBUG, `makeRequest, The server did not answer within [${requestTimeout}] seconds.`); - req.destroy(new Error("request timeout")); - }); - req.on("error", (err) => { - cc.log(cc.logLevelEnums.DEBUG, "makeRequest, Connection failed. Error:", err); + if (requestMethod === "POST") { + // write data to request body + req.write(compressed || body); + } + + req.end(); + } + catch (e) { + cc.log(cc.logLevelEnums.ERROR, "makeRequest, Failed the HTTP request. Error:", e); finish(true); - }); - if (method === "POST") { - // write data to request body - req.write(data); } + } + } + + /** + * Tells whether a request body may be sent gzip-compressed now: request compression is on, the server the request + * goes to said in its SDK behavior settings answer that it reads compressed bodies, and no compressed request was + * refused since that answer + * @param {String} url - server URL the request goes to + * @returns {Boolean} true when the body may be compressed + */ + function compressionAllowed(url) { + return requestCompressionEnabled && !requestCompressionBlocked && gzipAcceptingUrl !== null && gzipAcceptingUrl === url; + } - req.end(); + /** + * Compresses a request body with gzip, on the thread pool instead of the main thread + * @param {String} body - url encoded request body, checksum included + * @param {Function} done - called with the gzip bytes, or with null when the body is smaller than 1 KB or larger than + * 1 MB, compressing fails, or the result is not smaller than the body + */ + function compressRequestBody(body, done) { + var plain = Buffer.from(body, "utf8"); + if (plain.length < REQUEST_COMPRESSION_MIN_BYTES || plain.length > REQUEST_COMPRESSION_MAX_BYTES) { + done(null); + return; } - catch (e) { - // fallback - cc.log(cc.logLevelEnums.ERROR, "makeRequest, Failed the HTTP request. Error:", e); - finish(true); + zlib.gzip(plain, (err, compressed) => { + if (err || compressed.length >= plain.length) { + done(null); + return; + } + cc.log(cc.logLevelEnums.DEBUG, `makeRequest, The request body is sent gzip-compressed, [${plain.length}] -> [${compressed.length}] bytes.`); + done(compressed); + }); + } + + /** + * Stops request compression after the server, or something in front of it, answered a compressed request without + * accepting it, until the next SDK behavior settings answer says that the server reads compressed bodies, or the SDK + * starts again + */ + function blockRequestCompression() { + if (!requestCompressionBlocked) { + requestCompressionBlocked = true; + cc.log(cc.logLevelEnums.INFO, "makeRequest, A compressed request was refused, requests are sent uncompressed until a behavior settings answer says that the server reads compressed bodies, or the SDK starts again."); } } diff --git a/test/tests_request_compression.js b/test/tests_request_compression.js new file mode 100644 index 0000000..a940ad7 --- /dev/null +++ b/test/tests_request_compression.js @@ -0,0 +1,322 @@ +const assert = require("assert"); +const crypto = require("crypto"); +const zlib = require("zlib"); +const Countly = require("../lib/countly"); +const CountlyStorage = require("../lib/countly-storage"); +const behavior = require("../lib/countly-behavior"); +const hp = require("./helpers/helper_functions"); +const { startMockServer } = require("./helpers/mock_server"); +const { makeTempDir, removeTempDirs } = require("./helpers/run_script"); +const { appKey, settingsServer, settingsRequests, startSdk, storedValue, waitUntil } = require("./helpers/behavior_settings"); + +const DEVICE_ID = "gzip-user"; +const MARKER_QUERY = "?content_encoding=gzip"; + +/** + * Reads a recorded request the way the server does: the parameters of the URL and of the form body, the body + * decompressed when the URL carries the compression marker as its only query + * @param {Object} entry - request recorded by the mock server + * @returns {Object} compressed, params, body (the form body as text), bodyBytes (its size on the wire) and headers + */ +function read(entry) { + const url = new URL(entry.url, "http://localhost"); + const compressed = url.search === MARKER_QUERY; + const params = Object.fromEntries(url.searchParams); + delete params.content_encoding; + let body = ""; + if (entry.method === "POST") { + body = (compressed ? zlib.gunzipSync(entry.body) : entry.body).toString("utf8"); + Object.assign(params, Object.fromEntries(new URLSearchParams(body))); + } + return { compressed: compressed, params: params, body: body, bodyBytes: entry.body.length, headers: entry.headers }; +} + +/** + * The requests with the given marker parameter the server received, read with read() + * @param {Object} server - mock server + * @param {String} marker - value of the marker parameter + * @returns {Array} the requests + */ +function withMarker(server, marker) { + return server.requests.map(read).filter((r) => r.params.marker === marker); +} + +/** + * Queues a raw request with a payload of the given length, told apart by its marker parameter + * @param {String} marker - value of the marker parameter + * @param {Number} payloadLength - length of the payload parameter + */ +function queueRequest(marker, payloadLength) { + Countly.request({ app_key: appKey, device_id: DEVICE_ID, marker: marker, payload: "a".repeat(payloadLength) }); +} + +/** + * A settings answer, with "gzip": true when the server reads compressed bodies. Each answer carries its own "lkl" + * value, so the test can wait until the SDK has read it. + * @param {Number} lkl - key length limit that tells the answer apart + * @param {Boolean} gzip - whether the answer says the server reads compressed bodies + * @returns {Object} the answer + */ +function settingsAnswer(lkl, gzip) { + const answer = { v: 2, t: 1, c: { lkl: lkl } }; + if (gzip) { + answer.gzip = true; + } + return answer; +} + +/** + * Starts a mock server that answers settings requests with state.body, refuses the next state.refuse compressed + * requests with 400, drops the connection of the next state.drop compressed requests, answers remote config + * downloads with state.remoteConfig, and every other request with Success + * @param {Object} state - the answers + * @returns {Promise} the mock server + */ +function compressionServer(state) { + return settingsServer(state, (req, res, entry) => { + const request = read(entry); + if (request.compressed && state.refuse > 0) { + state.refuse--; + res.writeHead(400, { "Content-Type": "application/json" }); + res.end('{"result":"Invalid request body"}'); + return; + } + if (request.compressed && state.drop > 0) { + state.drop--; + req.socket.destroy(); + return; + } + if (request.params.method === "fetch_remote_config") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(state.remoteConfig); + return; + } + res.writeHead(200, { "Content-Type": "application/json" }); + res.end('{"result":"Success"}'); + }); +} + +/** + * Starts the SDK against the server and waits until it has read the first settings answer + * @param {Object} server - mock server + * @param {Object} state - its answers; state.body must carry "lkl" + * @param {Object} config - init options + */ +async function startAndReadSettings(server, state, config) { + startSdk(server, makeTempDir("gzip"), { device_id: DEVICE_ID, disable_health_check: true, ...config }); + assert.ok(await storedValue("cly_sbs", state.body.c), "the settings answer was not read"); +} + +/** + * Makes the SDK fetch the settings again, by a device ID change without merge, and waits until it has read the answer + * @param {Object} state - the server's answers + * @param {Object} answer - the next settings answer + * @param {String} deviceId - the new device ID + */ +async function fetchSettingsAgain(state, answer, deviceId) { + state.body = answer; + Countly.change_id(deviceId, false); + assert.ok(await storedValue("cly_sbs", answer.c), "the new settings answer was not read"); +} + +describe("Request compression tests", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); + afterEach(() => { + Countly.halt(true); + }); + after(() => { + removeTempDirs(); + }); + + it("1- without enable_request_compression does not ask for gzip and sends plain bodies", async() => { + const state = { body: settingsAnswer(101, true) }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, {}); + queueRequest("big", 5000); + const delivered = await Countly.flush(5); + await server.close(); + assert.strictEqual(delivered, true); + assert.strictEqual(settingsRequests(server)[0].params.gzip, undefined); + const sent = withMarker(server, "big"); + assert.equal(sent.length, 1); + assert.strictEqual(sent[0].compressed, false); + assert.equal(sent[0].params.payload.length, 5000); + }); + + it("2- sends a body of at least 1 KB gzip-compressed to the marked URL once the server says it reads gzip, with the checksum of the plain body", async() => { + const state = { body: settingsAnswer(101, true) }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true, salt: "gzip-salt" }); + queueRequest("big", 5000); + const delivered = await Countly.flush(5); + await server.close(); + assert.strictEqual(delivered, true); + assert.strictEqual(settingsRequests(server)[0].params.gzip, "1"); + const sent = withMarker(server, "big"); + assert.equal(sent.length, 1); + assert.strictEqual(sent[0].compressed, true); + assert.ok(sent[0].bodyBytes < 1000, `the compressed body has ${sent[0].bodyBytes} bytes`); + assert.equal(sent[0].headers["content-type"], "application/x-www-form-urlencoded"); + assert.strictEqual(sent[0].headers["content-encoding"], undefined); + assert.equal(sent[0].params.payload, "a".repeat(5000)); + const [signed, checksum] = sent[0].body.split("&checksum256="); + assert.equal(crypto.createHash("sha256").update(`${signed}gzip-salt`).digest("hex"), checksum); + assert.deepStrictEqual(CountlyStorage.storeGet("cly_sbs", null), { lkl: 101 }, "the gzip key was stored with the settings"); + }); + + it("3- keeps a body under 1 KB plain and compresses a bigger one after it", async() => { + const state = { body: settingsAnswer(101, true) }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true, force_post: true }); + queueRequest("small", 300); + queueRequest("big", 5000); + const delivered = await Countly.flush(5); + await server.close(); + assert.strictEqual(delivered, true); + const small = withMarker(server, "small"); + assert.equal(small.length, 1); + assert.strictEqual(small[0].compressed, false); + assert.ok(small[0].bodyBytes < 1024 && small[0].body.length > 0, "the small request was not sent as a POST body"); + assert.strictEqual(withMarker(server, "big")[0].compressed, true); + }); + + it("4- sends plain bodies when the settings answer has no gzip key", async() => { + const state = { body: settingsAnswer(101, false) }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true }); + queueRequest("big", 5000); + const delivered = await Countly.flush(5); + await server.close(); + assert.strictEqual(delivered, true); + assert.strictEqual(settingsRequests(server)[0].params.gzip, "1"); + assert.strictEqual(withMarker(server, "big")[0].compressed, false); + }); + + it("5- keeps a body over 1 MB plain and keeps compressing the requests after it", async() => { + const state = { body: settingsAnswer(101, true) }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true }); + queueRequest("huge", 1048576); + queueRequest("big", 5000); + const delivered = await Countly.flush(10); + await server.close(); + assert.strictEqual(delivered, true); + const huge = withMarker(server, "huge"); + assert.equal(huge.length, 1); + assert.strictEqual(huge[0].compressed, false); + assert.ok(huge[0].bodyBytes > 1048576); + assert.strictEqual(withMarker(server, "big")[0].compressed, true); + }); + + it("6- sends a refused compressed request again plain, and the requests after it plain", async() => { + const state = { body: settingsAnswer(101, true), refuse: 1 }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true }); + queueRequest("refused", 5000); + const first = await Countly.flush(5); + const second = await Countly.flush(5); + queueRequest("after", 5000); + const third = await Countly.flush(5); + await server.close(); + assert.deepStrictEqual([first, second, third], [false, true, true]); + const refused = withMarker(server, "refused"); + assert.deepStrictEqual(refused.map((r) => r.compressed), [true, false]); + assert.deepStrictEqual(refused[0].params, refused[1].params); + assert.deepStrictEqual(withMarker(server, "after").map((r) => r.compressed), [false]); + }); + + it("7- compresses again once a later settings answer says gzip, after a refusal stopped compression", async() => { + const state = { body: settingsAnswer(101, true), refuse: 1 }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true }); + queueRequest("refused", 5000); + await Countly.flush(5); + await Countly.flush(5); + + await fetchSettingsAgain(state, settingsAnswer(102, false), "gzip-user-2"); + queueRequest("without_gzip_answer", 5000); + await Countly.flush(5); + + await fetchSettingsAgain(state, settingsAnswer(103, true), "gzip-user-3"); + queueRequest("with_gzip_answer", 5000); + const delivered = await Countly.flush(5); + await server.close(); + assert.strictEqual(delivered, true); + assert.deepStrictEqual(settingsRequests(server).map((r) => r.params.gzip), ["1", "1", "1"]); + assert.deepStrictEqual(withMarker(server, "refused").map((r) => r.compressed), [true, false]); + assert.deepStrictEqual(withMarker(server, "without_gzip_answer").map((r) => r.compressed), [false]); + assert.deepStrictEqual(withMarker(server, "with_gzip_answer").map((r) => r.compressed), [true]); + }); + + [["JSON", '{"flag":"on"}', true], ["a page", "Bad gateway", false], ["nothing", "", false]].forEach(([kind, answer, staysOn], index) => { + it(`${8 + index}- ${staysOn ? "keeps" : "stops"} compression when a compressed remote config download is answered with ${kind}`, async() => { + const state = { body: settingsAnswer(101, true), remoteConfig: answer }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true, force_post: true }); + const keys = Array.from({ length: 120 }, (_, i) => `remote_config_key_${i}`); + await new Promise((resolve) => { + Countly.fetch_remote_config(keys, resolve); + }); + queueRequest("after_download", 5000); + const delivered = await Countly.flush(5); + await server.close(); + assert.strictEqual(delivered, true); + const download = server.requests.map(read).filter((r) => r.params.method === "fetch_remote_config"); + assert.equal(download.length, 1); + assert.strictEqual(download[0].compressed, true); + assert.equal(JSON.parse(download[0].params.keys).length, 120); + assert.deepStrictEqual(withMarker(server, "after_download").map((r) => r.compressed), [staysOn]); + }); + }); + + it("11- keeps compressing after a compressed request failed without an answer", async() => { + const state = { body: settingsAnswer(101, true), drop: 1 }; + const server = await compressionServer(state); + await startAndReadSettings(server, state, { enable_request_compression: true }); + queueRequest("dropped", 5000); + const first = await Countly.flush(5); + const second = await Countly.flush(5); + await server.close(); + assert.deepStrictEqual([first, second], [false, true]); + assert.deepStrictEqual(withMarker(server, "dropped").map((r) => r.compressed), [true, true]); + }); + + it("12- sends plain bodies after change_server_url until the new server says it reads gzip", async() => { + const state = { body: settingsAnswer(101, true) }; + const first = await compressionServer(state); + const held = []; + const second = await startMockServer((req, res, entry) => { + if (entry.params.method === "sc") { + held.push(res); + return; + } + res.writeHead(200, { "Content-Type": "application/json" }); + res.end('{"result":"Success"}'); + }); + await startAndReadSettings(first, state, { enable_request_compression: true }); + Countly.change_server_url(second.url); + assert.ok(await waitUntil(() => held.length === 1, 5000), "the new server was not asked for its settings"); + queueRequest("new_server", 5000); + const delivered = await Countly.flush(5); + held[0].writeHead(200, { "Content-Type": "application/json" }); + held[0].end(JSON.stringify(settingsAnswer(102, true))); + assert.ok(await storedValue("cly_sbs", { lkl: 102 }), "the new server's settings answer was not read"); + queueRequest("after_new_answer", 5000); + const deliveredAfter = await Countly.flush(5); + await first.close(); + await second.close(); + assert.deepStrictEqual([delivered, deliveredAfter], [true, true]); + assert.equal(withMarker(first, "new_server").length, 0); + assert.deepStrictEqual(withMarker(second, "new_server").map((r) => r.compressed), [false]); + assert.deepStrictEqual(withMarker(second, "after_new_answer").map((r) => r.compressed), [true]); + }); + + it("13- reads the gzip key of a settings answer only when it is true", () => { + assert.strictEqual(behavior.readsCompressedBodies(JSON.stringify({ v: 2, t: 1, c: {}, gzip: true })), true); + [JSON.stringify({ c: {}, gzip: "true" }), JSON.stringify({ c: {}, gzip: 1 }), JSON.stringify({ c: {}, gzip: false }), JSON.stringify({ c: { gzip: true } }), JSON.stringify([{ gzip: true }]), "null", "", "not json"].forEach((answer) => { + assert.strictEqual(behavior.readsCompressedBodies(answer), false, answer); + }); + }); +});