From a89f1fdde16f210dd61db32d43bacf36552b033e Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:50:17 +0900 Subject: [PATCH 1/2] fix: apply the internal limits to bulk events CountlyBulk read its limits from properties that were never set, so the events it queued were never cut. add_user now copies the bulk user's settings as given, since the event limits do not apply to them. --- CHANGELOG.md | 1 + lib/countly-bulk-user.js | 2 +- lib/countly-bulk.js | 8 +-- lib/countly.js | 2 +- test/helpers/scripts/bulk_cluster_limits.js | 23 ++++++ test/tests_bulk_fixes.js | 77 +++++++++++++++++++++ 6 files changed, 107 insertions(+), 6 deletions(-) create mode 100644 test/helpers/scripts/bulk_cluster_limits.js diff --git a/CHANGELOG.md b/CHANGELOG.md index f926a7a..aae32a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Mitigated an issue where every call without consent was logged as an error - Mitigated an issue where `add_consent` and `remove_consent` threw an error when called without a feature - Mitigated an issue where `add_event` and `end_event` threw an error when called without an event +- Mitigated an issue where events added to `CountlyBulk` ignored its `max_key_length`, `max_value_size` and `max_segmentation_values` limits ## 26.8.0 - ! Minor breaking change ! SDK behavior settings are now enabled by default. Changes made in SDK Manager on your server affect the SDK's behavior directly and take precedence over the matching init options: diff --git a/lib/countly-bulk-user.js b/lib/countly-bulk-user.js index a7d0f11..0cb6582 100644 --- a/lib/countly-bulk-user.js +++ b/lib/countly-bulk-user.js @@ -28,7 +28,7 @@ var cc = require("./countly-common"); * @param {boolean} [conf.require_consent=false] - Pass true if you are implementing GDPR compatible consent management. It would prevent running any functionality without proper consent * @param {number} [conf.max_key_length=128] - maximum size of all string keys * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) - * @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event + * @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 16ce0f4..8a6c1f0 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -46,7 +46,7 @@ CountlyBulk.StorageTypes = cc.storageTypeEnums; * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request * @param {number} [conf.max_key_length=128] - maximum size of all string keys * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) - * @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event + * @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length @@ -252,8 +252,8 @@ function CountlyBulk(conf) { if (!event.count) { event.count = 1; } - event.key = cc.truncateSingleValue(event.key, self.maxKeyLength, "add_event", self.debug); - event.segmentation = cc.truncateObject(event.segmentation, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_event", self.debug); + event.key = cc.truncateSingleValue(event.key, conf.maxKeyLength, "add_event"); + event.segmentation = cc.truncateObject(event.segmentation, conf.maxKeyLength, conf.maxValueSize, conf.maxSegmentationValues, "add_event"); var props = ["key", "count", "sum", "dur", "segmentation", "timestamp"]; var e = cc.makeJsonSafe(cc.getProperties(event, props), "add_event"); e.timestamp = e.timestamp || getMsTimestamp(); @@ -352,7 +352,7 @@ function CountlyBulk(conf) { * var user = server.add_user({device_id:"my_device_id"}); * */ this.add_user = function(userConf) { - userConf = cc.truncateObject(userConf, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_user", self.debug); + userConf = cc.makeJsonSafe(userConf, "add_user") || {}; userConf.server = this; return new BulkUser(userConf); }; diff --git a/lib/countly.js b/lib/countly.js index 0a328ec..854dabb 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -155,7 +155,7 @@ Countly.Bulk = Bulk; * @deprecated {number} [conf.max_logs=100] - maximum amount of breadcrumbs to store for crash logs * @param {number} [conf.max_key_length=128] - maximum size of all string keys * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) - * @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event + * @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length diff --git a/test/helpers/scripts/bulk_cluster_limits.js b/test/helpers/scripts/bulk_cluster_limits.js new file mode 100644 index 0000000..9771788 --- /dev/null +++ b/test/helpers/scripts/bulk_cluster_limits.js @@ -0,0 +1,23 @@ +const cluster = require("cluster"); +const CountlyBulk = require("../../../lib/countly-bulk"); + +const server = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: process.env.COUNTLY_URL, + interval: 200, + max_key_length: 4, + max_value_size: 5, +}); + +if (cluster.isMaster) { + server.start(); + const worker = cluster.fork(); + setTimeout(() => { + worker.kill(); + process.exit(0); + }, Number(process.env.RUN_MS || 8000)); +} +else { + server.add_event("cluster-limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value" } }); + setInterval(() => { }, 1000); +} diff --git a/test/tests_bulk_fixes.js b/test/tests_bulk_fixes.js index ccf9076..d244166 100644 --- a/test/tests_bulk_fixes.js +++ b/test/tests_bulk_fixes.js @@ -185,4 +185,81 @@ describe("Bulk fix tests", () => { await server.close(); assert.ok(resent, "the bulk queue stopped after the cut-off response"); }); + + it("10- events reach the server cut to the internal limits set on the bulk instance", async() => { + const server = await startMockServer(); + const bulk = new CountlyBulk({ app_key: appKey, url: server.url, interval: 100, max_key_length: 4, max_value_size: 5, max_segmentation_values: 2 }); + bulk.add_event("limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value", plan: "pro", coupon: "dropped" } }); + bulk.start(); + const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "limits-user"), 10000); + bulk.stop(); + await server.close(); + assert.ok(sent, "the event was not sent"); + const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "limits-user")[0]; + assert.equal(events.length, 1); + assert.equal(events[0].key, "chec"); + assert.deepStrictEqual(events[0].segmentation, { cate: "overl", plan: "pro" }); + }); + + it("11- events added to the bulk instance keep to the default internal limits", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1" }); + const segmentation = {}; + for (let i = 0; i < 101; i++) { + segmentation[`${i}-${"k".repeat(200)}`] = "v".repeat(300); + } + bulk.add_event("default-limits-user", { key: "e".repeat(200), segmentation: segmentation }); + const event = bulk._getBulkEventQueue()["default-limits-user"][0]; + assert.equal(event.key, "e".repeat(128)); + const keys = Object.keys(event.segmentation); + assert.equal(keys.length, 100); + keys.forEach((key, i) => { + assert.equal(key, `${i}-${"k".repeat(200)}`.substring(0, 128)); + assert.equal(event.segmentation[key], "v".repeat(256)); + }); + }); + + it("12- events of a bulk user keep to the internal limits set on the bulk instance", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5 }); + bulk.add_user({ device_id: "limits-bulk-user" }).add_event({ key: "checkout_completed", segmentation: { category: "overlong-value" } }); + const events = bulk._getBulkEventQueue()["limits-bulk-user"]; + assert.equal(events.length, 1); + assert.equal(events[0].key, "chec"); + assert.deepStrictEqual(events[0].segmentation, { cate: "overl" }); + }); + + it("13- events that a cluster worker hands to the main process keep to the main process's internal limits", async() => { + const server = await startMockServer(); + const run = runScript("bulk_cluster_limits.js", { COUNTLY_URL: server.url, RUN_MS: "8000" }, 30000); + const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "cluster-limits-user"), 20000); + const result = await run; + await server.close(); + assert.ok(sent, `the worker's event was not sent: ${result.stderr}`); + const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "cluster-limits-user")[0]; + assert.equal(events.length, 1); + assert.equal(events[0].key, "chec"); + assert.deepStrictEqual(events[0].segmentation, { cate: "overl" }); + }); + + it("14- a bulk user keeps its settings whole when the bulk instance has low internal limits", () => { + // the limits are shorter than the setting names and values, and allow one setting only + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 8, max_value_size: 5, max_segmentation_values: 1 }); + const settings = { ip_address: "203.0.113.7", country_code: "TR", city: "Istanbul", require_consent: true }; + const withoutConsent = bulk.add_user({ device_id: "settings-user-without-consent", ...settings }); + withoutConsent.begin_session(); + withoutConsent.add_event({ key: "purchase" }); + const withConsent = bulk.add_user({ device_id: "settings-user-with-consent", ...settings }); + withConsent.add_consent(["sessions", "location", "events"]); + withConsent.begin_session(); + withConsent.add_event({ key: "purchase" }); + + const requests = bulk._getBulkRequestQueue(); + assert.equal(requests.filter((r) => r.device_id === "settings-user-without-consent").length, 0, "a request of the user without consent was queued"); + assert.equal(bulk._getBulkEventQueue()["settings-user-without-consent"], undefined, "an event of the user without consent was queued"); + const session = requests.find((r) => r.device_id === "settings-user-with-consent" && r.begin_session === 1); + assert.ok(session, "the session of the user with consent was not queued"); + assert.equal(session.ip_address, "203.0.113.7"); + assert.equal(session.country_code, "TR"); + assert.equal(session.city, "Istanbul"); + assert.equal(bulk._getBulkEventQueue()["settings-user-with-consent"].length, 1); + }); }); From 135d896c9f79477b28317c4616e0a5373cda602f Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:10:38 +0900 Subject: [PATCH 2/2] fix: let bulk users use the limits set for the bulk instance A bulk user took its limits only from its own settings and otherwise used the defaults, so the limits set for CountlyBulk did not reach user details, custom properties, views, conversions and traces. A limit set for the user still takes precedence. --- CHANGELOG.md | 1 + lib/countly-bulk-user.js | 25 +++++++++++++------------ test/tests_bulk_fixes.js | 23 +++++++++++++++++++++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae32a6..ad9ddf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Mitigated an issue where `add_consent` and `remove_consent` threw an error when called without a feature - Mitigated an issue where `add_event` and `end_event` threw an error when called without an event - Mitigated an issue where events added to `CountlyBulk` ignored its `max_key_length`, `max_value_size` and `max_segmentation_values` limits +- Mitigated an issue where bulk users ignored the limits set for `CountlyBulk` and used the default limits instead ## 26.8.0 - ! Minor breaking change ! SDK behavior settings are now enabled by default. Changes made in SDK Manager on your server affect the SDK's behavior directly and take precedence over the matching init options: diff --git a/lib/countly-bulk-user.js b/lib/countly-bulk-user.js index 0cb6582..337718d 100644 --- a/lib/countly-bulk-user.js +++ b/lib/countly-bulk-user.js @@ -26,12 +26,12 @@ var cc = require("./countly-common"); * @param {string=} conf.ip_address - ip address of your user * @param {boolean} [conf.debug=false] - output debug info into console * @param {boolean} [conf.require_consent=false] - Pass true if you are implementing GDPR compatible consent management. It would prevent running any functionality without proper consent - * @param {number} [conf.max_key_length=128] - maximum size of all string keys - * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) - * @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event - * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted - * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread - * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length + * @param {number} [conf.max_key_length] - maximum size of all string keys. Defaults to the limit of the CountlyBulk instance, 128 unless set there + * @param {number} [conf.max_value_size] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars). Defaults to the limit of the CountlyBulk instance, 256 unless set there + * @param {number} [conf.max_segmentation_values] - max amount of custom (dev provided) segmentation in one event. Defaults to the limit of the CountlyBulk instance, 100 unless set there + * @param {number} [conf.max_breadcrumb_count] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted. Defaults to the limit of the CountlyBulk instance, 100 unless set there + * @param {number} [conf.max_stack_trace_lines_per_thread] - maximum amount of stack trace lines would be recorded per thread. Defaults to the limit of the CountlyBulk instance, 30 unless set there + * @param {number} [conf.max_stack_trace_line_length] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length. Defaults to the limit of the CountlyBulk instance, 200 unless set there * @example * var CountlyBulk = require("countly-sdk-nodejs").Bulk; * @@ -80,12 +80,13 @@ function CountlyBulkUser(conf) { } var self = this; - conf.maxKeyLength = conf.max_key_length || maxKeyLength; - conf.maxValueSize = conf.max_value_size || maxValueSize; - conf.maxSegmentationValues = conf.max_segmentation_values || maxSegmentationValues; - conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount; - conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; - conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; + var serverConf = conf.server.conf || {}; + conf.maxKeyLength = conf.max_key_length || serverConf.maxKeyLength || maxKeyLength; + conf.maxValueSize = conf.max_value_size || serverConf.maxValueSize || maxValueSize; + conf.maxSegmentationValues = conf.max_segmentation_values || serverConf.maxSegmentationValues || maxSegmentationValues; + conf.maxBreadcrumbCount = conf.max_breadcrumb_count || serverConf.maxBreadcrumbCount || maxBreadcrumbCount; + conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || serverConf.maxStackTraceLinesPerThread || maxStackTraceLinesPerThread; + conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || serverConf.maxStackTraceLineLength || maxStackTraceLineLength; /** * Modify feature groups for consent management. Allows you to group multiple features under one feature group diff --git a/test/tests_bulk_fixes.js b/test/tests_bulk_fixes.js index d244166..906662e 100644 --- a/test/tests_bulk_fixes.js +++ b/test/tests_bulk_fixes.js @@ -262,4 +262,27 @@ describe("Bulk fix tests", () => { assert.equal(session.city, "Istanbul"); assert.equal(bulk._getBulkEventQueue()["settings-user-with-consent"].length, 1); }); + + it("15- a bulk user's profile, custom properties, views and traces keep to the internal limits set on the bulk instance", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5, max_segmentation_values: 1 }); + const user = bulk.add_user({ device_id: "instance-limits-user" }); + user.user_details({ name: "Long name", custom: { plan: "enterprise", team: "dropped" } }); + user.custom_set("department", "engineering").custom_save(); + user.report_view("Checkout page", 1700000000, 5); + user.report_trace({ type: "device", name: "app_start", stz: 1700000000000, etz: 1700000001000, apm_metrics: { duration: 1000 } }); + + const requests = bulk._getBulkRequestQueue().filter((r) => r.device_id === "instance-limits-user"); + assert.equal(requests.length, 4); + assert.deepStrictEqual(requests.find((r) => r.user_details && r.user_details.name).user_details, { name: "Long ", custom: { plan: "enter" } }); + assert.deepStrictEqual(requests.find((r) => r.user_details && !r.user_details.name).user_details, { custom: { depa: "engin" } }); + assert.equal(requests.find((r) => r.events).events[0].segmentation.name, "Check"); + assert.equal(JSON.parse(requests.find((r) => r.apm).apm).name, "app_"); + }); + + it("16- a bulk user's own limits take precedence over the ones set on the bulk instance", () => { + const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_value_size: 5 }); + bulk.add_user({ device_id: "own-limits-user", max_value_size: 8 }).user_details({ name: "Long name here" }); + const request = bulk._getBulkRequestQueue().find((r) => r.device_id === "own-limits-user"); + assert.deepStrictEqual(request.user_details, { name: "Long nam", custom: {} }); + }); });