Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
- Mitigated an issue where every call without consent was logged as an error
- Mitigated an issue where `add_consent` and `remove_consent` threw an error when called without a feature
- Mitigated an issue where `add_event` and `end_event` threw an error when called without an event
- Mitigated an issue where events added to `CountlyBulk` ignored its `max_key_length`, `max_value_size` and `max_segmentation_values` limits
- Mitigated an issue where bulk users ignored the limits set for `CountlyBulk` and used the default limits instead

## 26.8.0
- ! Minor breaking change ! SDK behavior settings are now enabled by default. Changes made in SDK Manager on your server affect the SDK's behavior directly and take precedence over the matching init options:
Expand Down
25 changes: 13 additions & 12 deletions lib/countly-bulk-user.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ var cc = require("./countly-common");
* @param {string=} conf.ip_address - ip address of your user
* @param {boolean} [conf.debug=false] - output debug info into console
* @param {boolean} [conf.require_consent=false] - Pass true if you are implementing GDPR compatible consent management. It would prevent running any functionality without proper consent
* @param {number} [conf.max_key_length=128] - maximum size of all string keys
* @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars)
* @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
* @param {number} [conf.max_key_length] - maximum size of all string keys. Defaults to the limit of the CountlyBulk instance, 128 unless set there
* @param {number} [conf.max_value_size] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars). Defaults to the limit of the CountlyBulk instance, 256 unless set there
* @param {number} [conf.max_segmentation_values] - max amount of custom (dev provided) segmentation in one event. Defaults to the limit of the CountlyBulk instance, 100 unless set there
* @param {number} [conf.max_breadcrumb_count] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted. Defaults to the limit of the CountlyBulk instance, 100 unless set there
* @param {number} [conf.max_stack_trace_lines_per_thread] - maximum amount of stack trace lines would be recorded per thread. Defaults to the limit of the CountlyBulk instance, 30 unless set there
* @param {number} [conf.max_stack_trace_line_length] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length. Defaults to the limit of the CountlyBulk instance, 200 unless set there
* @example
* var CountlyBulk = require("countly-sdk-nodejs").Bulk;
*
Expand Down Expand Up @@ -80,12 +80,13 @@ function CountlyBulkUser(conf) {
}

var self = this;
conf.maxKeyLength = conf.max_key_length || maxKeyLength;
conf.maxValueSize = conf.max_value_size || maxValueSize;
conf.maxSegmentationValues = conf.max_segmentation_values || maxSegmentationValues;
conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount;
conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength;
var serverConf = conf.server.conf || {};
conf.maxKeyLength = conf.max_key_length || serverConf.maxKeyLength || maxKeyLength;
conf.maxValueSize = conf.max_value_size || serverConf.maxValueSize || maxValueSize;
conf.maxSegmentationValues = conf.max_segmentation_values || serverConf.maxSegmentationValues || maxSegmentationValues;
conf.maxBreadcrumbCount = conf.max_breadcrumb_count || serverConf.maxBreadcrumbCount || maxBreadcrumbCount;
conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || serverConf.maxStackTraceLinesPerThread || maxStackTraceLinesPerThread;
conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || serverConf.maxStackTraceLineLength || maxStackTraceLineLength;

/**
* Modify feature groups for consent management. Allows you to group multiple features under one feature group
Expand Down
8 changes: 4 additions & 4 deletions lib/countly-bulk.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ CountlyBulk.StorageTypes = cc.storageTypeEnums;
* @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request
* @param {number} [conf.max_key_length=128] - maximum size of all string keys
* @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars)
* @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
Expand Down Expand Up @@ -252,8 +252,8 @@ function CountlyBulk(conf) {
if (!event.count) {
event.count = 1;
}
event.key = cc.truncateSingleValue(event.key, self.maxKeyLength, "add_event", self.debug);
event.segmentation = cc.truncateObject(event.segmentation, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_event", self.debug);
event.key = cc.truncateSingleValue(event.key, conf.maxKeyLength, "add_event");
event.segmentation = cc.truncateObject(event.segmentation, conf.maxKeyLength, conf.maxValueSize, conf.maxSegmentationValues, "add_event");
var props = ["key", "count", "sum", "dur", "segmentation", "timestamp"];
var e = cc.makeJsonSafe(cc.getProperties(event, props), "add_event");
e.timestamp = e.timestamp || getMsTimestamp();
Expand Down Expand Up @@ -352,7 +352,7 @@ function CountlyBulk(conf) {
* var user = server.add_user({device_id:"my_device_id"});
* */
this.add_user = function(userConf) {
userConf = cc.truncateObject(userConf, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_user", self.debug);
userConf = cc.makeJsonSafe(userConf, "add_user") || {};
userConf.server = this;
return new BulkUser(userConf);
};
Expand Down
2 changes: 1 addition & 1 deletion lib/countly.js
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ Countly.Bulk = Bulk;
* @deprecated {number} [conf.max_logs=100] - maximum amount of breadcrumbs to store for crash logs
* @param {number} [conf.max_key_length=128] - maximum size of all string keys
* @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars)
* @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
Expand Down
23 changes: 23 additions & 0 deletions test/helpers/scripts/bulk_cluster_limits.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
const cluster = require("cluster");
const CountlyBulk = require("../../../lib/countly-bulk");

const server = new CountlyBulk({
app_key: "YOUR_APP_KEY",
url: process.env.COUNTLY_URL,
interval: 200,
max_key_length: 4,
max_value_size: 5,
});

if (cluster.isMaster) {
server.start();
const worker = cluster.fork();
setTimeout(() => {
worker.kill();
process.exit(0);
}, Number(process.env.RUN_MS || 8000));
}
else {
server.add_event("cluster-limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value" } });
setInterval(() => { }, 1000);
}
100 changes: 100 additions & 0 deletions test/tests_bulk_fixes.js
Original file line number Diff line number Diff line change
Expand Up @@ -185,4 +185,104 @@ describe("Bulk fix tests", () => {
await server.close();
assert.ok(resent, "the bulk queue stopped after the cut-off response");
});

it("10- events reach the server cut to the internal limits set on the bulk instance", async() => {
const server = await startMockServer();
const bulk = new CountlyBulk({ app_key: appKey, url: server.url, interval: 100, max_key_length: 4, max_value_size: 5, max_segmentation_values: 2 });
bulk.add_event("limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value", plan: "pro", coupon: "dropped" } });
bulk.start();
const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "limits-user"), 10000);
bulk.stop();
await server.close();
assert.ok(sent, "the event was not sent");
const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "limits-user")[0];
assert.equal(events.length, 1);
assert.equal(events[0].key, "chec");
assert.deepStrictEqual(events[0].segmentation, { cate: "overl", plan: "pro" });
});

it("11- events added to the bulk instance keep to the default internal limits", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1" });
const segmentation = {};
for (let i = 0; i < 101; i++) {
segmentation[`${i}-${"k".repeat(200)}`] = "v".repeat(300);
}
bulk.add_event("default-limits-user", { key: "e".repeat(200), segmentation: segmentation });
const event = bulk._getBulkEventQueue()["default-limits-user"][0];
assert.equal(event.key, "e".repeat(128));
const keys = Object.keys(event.segmentation);
assert.equal(keys.length, 100);
keys.forEach((key, i) => {
assert.equal(key, `${i}-${"k".repeat(200)}`.substring(0, 128));
assert.equal(event.segmentation[key], "v".repeat(256));
});
});

it("12- events of a bulk user keep to the internal limits set on the bulk instance", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5 });
bulk.add_user({ device_id: "limits-bulk-user" }).add_event({ key: "checkout_completed", segmentation: { category: "overlong-value" } });
const events = bulk._getBulkEventQueue()["limits-bulk-user"];
assert.equal(events.length, 1);
assert.equal(events[0].key, "chec");
assert.deepStrictEqual(events[0].segmentation, { cate: "overl" });
});

it("13- events that a cluster worker hands to the main process keep to the main process's internal limits", async() => {
const server = await startMockServer();
const run = runScript("bulk_cluster_limits.js", { COUNTLY_URL: server.url, RUN_MS: "8000" }, 30000);
const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "cluster-limits-user"), 20000);
const result = await run;
await server.close();
assert.ok(sent, `the worker's event was not sent: ${result.stderr}`);
const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "cluster-limits-user")[0];
assert.equal(events.length, 1);
assert.equal(events[0].key, "chec");
assert.deepStrictEqual(events[0].segmentation, { cate: "overl" });
});

it("14- a bulk user keeps its settings whole when the bulk instance has low internal limits", () => {
// the limits are shorter than the setting names and values, and allow one setting only
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 8, max_value_size: 5, max_segmentation_values: 1 });
const settings = { ip_address: "203.0.113.7", country_code: "TR", city: "Istanbul", require_consent: true };
const withoutConsent = bulk.add_user({ device_id: "settings-user-without-consent", ...settings });
withoutConsent.begin_session();
withoutConsent.add_event({ key: "purchase" });
const withConsent = bulk.add_user({ device_id: "settings-user-with-consent", ...settings });
withConsent.add_consent(["sessions", "location", "events"]);
withConsent.begin_session();
withConsent.add_event({ key: "purchase" });

const requests = bulk._getBulkRequestQueue();
assert.equal(requests.filter((r) => r.device_id === "settings-user-without-consent").length, 0, "a request of the user without consent was queued");
assert.equal(bulk._getBulkEventQueue()["settings-user-without-consent"], undefined, "an event of the user without consent was queued");
const session = requests.find((r) => r.device_id === "settings-user-with-consent" && r.begin_session === 1);
assert.ok(session, "the session of the user with consent was not queued");
assert.equal(session.ip_address, "203.0.113.7");
assert.equal(session.country_code, "TR");
assert.equal(session.city, "Istanbul");
assert.equal(bulk._getBulkEventQueue()["settings-user-with-consent"].length, 1);
});

it("15- a bulk user's profile, custom properties, views and traces keep to the internal limits set on the bulk instance", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5, max_segmentation_values: 1 });
const user = bulk.add_user({ device_id: "instance-limits-user" });
user.user_details({ name: "Long name", custom: { plan: "enterprise", team: "dropped" } });
user.custom_set("department", "engineering").custom_save();
user.report_view("Checkout page", 1700000000, 5);
user.report_trace({ type: "device", name: "app_start", stz: 1700000000000, etz: 1700000001000, apm_metrics: { duration: 1000 } });

const requests = bulk._getBulkRequestQueue().filter((r) => r.device_id === "instance-limits-user");
assert.equal(requests.length, 4);
assert.deepStrictEqual(requests.find((r) => r.user_details && r.user_details.name).user_details, { name: "Long ", custom: { plan: "enter" } });
assert.deepStrictEqual(requests.find((r) => r.user_details && !r.user_details.name).user_details, { custom: { depa: "engin" } });
assert.equal(requests.find((r) => r.events).events[0].segmentation.name, "Check");
assert.equal(JSON.parse(requests.find((r) => r.apm).apm).name, "app_");
});

it("16- a bulk user's own limits take precedence over the ones set on the bulk instance", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_value_size: 5 });
bulk.add_user({ device_id: "own-limits-user", max_value_size: 8 }).user_details({ name: "Long name here" });
const request = bulk._getBulkRequestQueue().find((r) => r.device_id === "own-limits-user");
assert.deepStrictEqual(request.user_details, { name: "Long nam", custom: {} });
});
});
Loading