diff --git a/CHANGELOG.md b/CHANGELOG.md index f926a7a..ad9ddf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - Mitigated an issue where every call without consent was logged as an error - Mitigated an issue where `add_consent` and `remove_consent` threw an error when called without a feature - Mitigated an issue where `add_event` and `end_event` threw an error when called without an event +- Mitigated an issue where events added to `CountlyBulk` ignored its `max_key_length`, `max_value_size` and `max_segmentation_values` limits +- Mitigated an issue where bulk users ignored the limits set for `CountlyBulk` and used the default limits instead ## 26.8.0 - ! Minor breaking change ! SDK behavior settings are now enabled by default. Changes made in SDK Manager on your server affect the SDK's behavior directly and take precedence over the matching init options: diff --git a/lib/countly-bulk-user.js b/lib/countly-bulk-user.js index a7d0f11..337718d 100644 --- a/lib/countly-bulk-user.js +++ b/lib/countly-bulk-user.js @@ -26,12 +26,12 @@ var cc = require("./countly-common"); * @param {string=} conf.ip_address - ip address of your user * @param {boolean} [conf.debug=false] - output debug info into console * @param {boolean} [conf.require_consent=false] - Pass true if you are implementing GDPR compatible consent management. It would prevent running any functionality without proper consent - * @param {number} [conf.max_key_length=128] - maximum size of all string keys - * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) - * @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event - * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted - * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread - * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length + * @param {number} [conf.max_key_length] - maximum size of all string keys. Defaults to the limit of the CountlyBulk instance, 128 unless set there + * @param {number} [conf.max_value_size] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars). Defaults to the limit of the CountlyBulk instance, 256 unless set there + * @param {number} [conf.max_segmentation_values] - max amount of custom (dev provided) segmentation in one event. Defaults to the limit of the CountlyBulk instance, 100 unless set there + * @param {number} [conf.max_breadcrumb_count] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted. Defaults to the limit of the CountlyBulk instance, 100 unless set there + * @param {number} [conf.max_stack_trace_lines_per_thread] - maximum amount of stack trace lines would be recorded per thread. Defaults to the limit of the CountlyBulk instance, 30 unless set there + * @param {number} [conf.max_stack_trace_line_length] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length. Defaults to the limit of the CountlyBulk instance, 200 unless set there * @example * var CountlyBulk = require("countly-sdk-nodejs").Bulk; * @@ -80,12 +80,13 @@ function CountlyBulkUser(conf) { } var self = this; - conf.maxKeyLength = conf.max_key_length || maxKeyLength; - conf.maxValueSize = conf.max_value_size || maxValueSize; - conf.maxSegmentationValues = conf.max_segmentation_values || maxSegmentationValues; - conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount; - conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; - conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; + var serverConf = conf.server.conf || {}; + conf.maxKeyLength = conf.max_key_length || serverConf.maxKeyLength || maxKeyLength; + conf.maxValueSize = conf.max_value_size || serverConf.maxValueSize || maxValueSize; + conf.maxSegmentationValues = conf.max_segmentation_values || serverConf.maxSegmentationValues || maxSegmentationValues; + conf.maxBreadcrumbCount = conf.max_breadcrumb_count || serverConf.maxBreadcrumbCount || maxBreadcrumbCount; + conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || serverConf.maxStackTraceLinesPerThread || maxStackTraceLinesPerThread; + conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || serverConf.maxStackTraceLineLength || maxStackTraceLineLength; /** * Modify feature groups for consent management. Allows you to group multiple features under one feature group diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 16ce0f4..8a6c1f0 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -46,7 +46,7 @@ CountlyBulk.StorageTypes = cc.storageTypeEnums; * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request * @param {number} [conf.max_key_length=128] - maximum size of all string keys * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) - * @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event + * @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length @@ -252,8 +252,8 @@ function CountlyBulk(conf) { if (!event.count) { event.count = 1; } - event.key = cc.truncateSingleValue(event.key, self.maxKeyLength, "add_event", self.debug); - event.segmentation = cc.truncateObject(event.segmentation, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_event", self.debug); + event.key = cc.truncateSingleValue(event.key, conf.maxKeyLength, "add_event"); + event.segmentation = cc.truncateObject(event.segmentation, conf.maxKeyLength, conf.maxValueSize, conf.maxSegmentationValues, "add_event"); var props = ["key", "count", "sum", "dur", "segmentation", "timestamp"]; var e = cc.makeJsonSafe(cc.getProperties(event, props), "add_event"); e.timestamp = e.timestamp || getMsTimestamp(); @@ -352,7 +352,7 @@ function CountlyBulk(conf) { * var user = server.add_user({device_id:"my_device_id"}); * */ this.add_user = function(userConf) { - userConf = cc.truncateObject(userConf, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_user", self.debug); + userConf = cc.makeJsonSafe(userConf, "add_user") || {}; userConf.server = this; return new BulkUser(userConf); }; diff --git a/lib/countly.js b/lib/countly.js index 0a328ec..854dabb 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -155,7 +155,7 @@ Countly.Bulk = Bulk; * @deprecated {number} [conf.max_logs=100] - maximum amount of breadcrumbs to store for crash logs * @param {number} [conf.max_key_length=128] - maximum size of all string keys * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) - * @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event + * @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length diff --git a/test/helpers/scripts/bulk_cluster_limits.js b/test/helpers/scripts/bulk_cluster_limits.js new file mode 100644 index 0000000..9771788 --- /dev/null +++ b/test/helpers/scripts/bulk_cluster_limits.js @@ -0,0 +1,23 @@ +const cluster = require("cluster"); +const CountlyBulk = require("../../../lib/countly-bulk"); + +const server = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: process.env.COUNTLY_URL, + interval: 200, + max_key_length: 4, + max_value_size: 5, +}); + +if (cluster.isMaster) { + server.start(); + const worker = cluster.fork(); + setTimeout(() => { + worker.kill(); + process.exit(0); + }, Number(process.env.RUN_MS || 8000)); +} +else { + server.add_event("cluster-limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value" } }); + setInterval(() => { }, 1000); +} diff --git a/test/tests_bulk_fixes.js b/test/tests_bulk_fixes.js index ccf9076..906662e 100644 --- a/test/tests_bulk_fixes.js +++ b/test/tests_bulk_fixes.js @@ -185,4 +185,104 @@ describe("Bulk fix tests", () => { await server.close(); assert.ok(resent, "the bulk queue stopped after the cut-off response"); }); + + it("10- events reach the server cut to the internal limits set on the bulk instance", async() => { + const server = await startMockServer(); + const bulk = new CountlyBulk({ app_key: appKey, url: server.url, interval: 100, max_key_length: 4, max_value_size: 5, max_segmentation_values: 2 }); + bulk.add_event("limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value", plan: "pro", coupon: "dropped" } }); + bulk.start(); + const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "limits-user"), 10000); + bulk.stop(); + await server.close(); + assert.ok(sent, "the event was not sent"); + const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "limits-user")[0]; + assert.equal(events.length, 1); + assert.equal(events[0].key, "chec"); + assert.deepStrictEqual(events[0].segmentation, { cate: "overl", plan: "pro" }); + }); + + it("11- events added to the bulk instance keep to the default internal limits", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1" }); + const segmentation = {}; + for (let i = 0; i < 101; i++) { + segmentation[`${i}-${"k".repeat(200)}`] = "v".repeat(300); + } + bulk.add_event("default-limits-user", { key: "e".repeat(200), segmentation: segmentation }); + const event = bulk._getBulkEventQueue()["default-limits-user"][0]; + assert.equal(event.key, "e".repeat(128)); + const keys = Object.keys(event.segmentation); + assert.equal(keys.length, 100); + keys.forEach((key, i) => { + assert.equal(key, `${i}-${"k".repeat(200)}`.substring(0, 128)); + assert.equal(event.segmentation[key], "v".repeat(256)); + }); + }); + + it("12- events of a bulk user keep to the internal limits set on the bulk instance", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5 }); + bulk.add_user({ device_id: "limits-bulk-user" }).add_event({ key: "checkout_completed", segmentation: { category: "overlong-value" } }); + const events = bulk._getBulkEventQueue()["limits-bulk-user"]; + assert.equal(events.length, 1); + assert.equal(events[0].key, "chec"); + assert.deepStrictEqual(events[0].segmentation, { cate: "overl" }); + }); + + it("13- events that a cluster worker hands to the main process keep to the main process's internal limits", async() => { + const server = await startMockServer(); + const run = runScript("bulk_cluster_limits.js", { COUNTLY_URL: server.url, RUN_MS: "8000" }, 30000); + const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "cluster-limits-user"), 20000); + const result = await run; + await server.close(); + assert.ok(sent, `the worker's event was not sent: ${result.stderr}`); + const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "cluster-limits-user")[0]; + assert.equal(events.length, 1); + assert.equal(events[0].key, "chec"); + assert.deepStrictEqual(events[0].segmentation, { cate: "overl" }); + }); + + it("14- a bulk user keeps its settings whole when the bulk instance has low internal limits", () => { + // the limits are shorter than the setting names and values, and allow one setting only + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 8, max_value_size: 5, max_segmentation_values: 1 }); + const settings = { ip_address: "203.0.113.7", country_code: "TR", city: "Istanbul", require_consent: true }; + const withoutConsent = bulk.add_user({ device_id: "settings-user-without-consent", ...settings }); + withoutConsent.begin_session(); + withoutConsent.add_event({ key: "purchase" }); + const withConsent = bulk.add_user({ device_id: "settings-user-with-consent", ...settings }); + withConsent.add_consent(["sessions", "location", "events"]); + withConsent.begin_session(); + withConsent.add_event({ key: "purchase" }); + + const requests = bulk._getBulkRequestQueue(); + assert.equal(requests.filter((r) => r.device_id === "settings-user-without-consent").length, 0, "a request of the user without consent was queued"); + assert.equal(bulk._getBulkEventQueue()["settings-user-without-consent"], undefined, "an event of the user without consent was queued"); + const session = requests.find((r) => r.device_id === "settings-user-with-consent" && r.begin_session === 1); + assert.ok(session, "the session of the user with consent was not queued"); + assert.equal(session.ip_address, "203.0.113.7"); + assert.equal(session.country_code, "TR"); + assert.equal(session.city, "Istanbul"); + assert.equal(bulk._getBulkEventQueue()["settings-user-with-consent"].length, 1); + }); + + it("15- a bulk user's profile, custom properties, views and traces keep to the internal limits set on the bulk instance", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5, max_segmentation_values: 1 }); + const user = bulk.add_user({ device_id: "instance-limits-user" }); + user.user_details({ name: "Long name", custom: { plan: "enterprise", team: "dropped" } }); + user.custom_set("department", "engineering").custom_save(); + user.report_view("Checkout page", 1700000000, 5); + user.report_trace({ type: "device", name: "app_start", stz: 1700000000000, etz: 1700000001000, apm_metrics: { duration: 1000 } }); + + const requests = bulk._getBulkRequestQueue().filter((r) => r.device_id === "instance-limits-user"); + assert.equal(requests.length, 4); + assert.deepStrictEqual(requests.find((r) => r.user_details && r.user_details.name).user_details, { name: "Long ", custom: { plan: "enter" } }); + assert.deepStrictEqual(requests.find((r) => r.user_details && !r.user_details.name).user_details, { custom: { depa: "engin" } }); + assert.equal(requests.find((r) => r.events).events[0].segmentation.name, "Check"); + assert.equal(JSON.parse(requests.find((r) => r.apm).apm).name, "app_"); + }); + + it("16- a bulk user's own limits take precedence over the ones set on the bulk instance", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_value_size: 5 }); + bulk.add_user({ device_id: "own-limits-user", max_value_size: 8 }).user_details({ name: "Long name here" }); + const request = bulk._getBulkRequestQueue().find((r) => r.device_id === "own-limits-user"); + assert.deepStrictEqual(request.user_details, { name: "Long nam", custom: {} }); + }); });