Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
- Mitigated an issue where every call without consent was logged as an error
- Mitigated an issue where `add_consent` and `remove_consent` threw an error when called without a feature
- Mitigated an issue where `add_event` and `end_event` threw an error when called without an event
- Mitigated an issue where events added to `CountlyBulk` ignored its `max_key_length`, `max_value_size` and `max_segmentation_values` limits

## 26.8.0
- ! Minor breaking change ! SDK behavior settings are now enabled by default. Changes made in SDK Manager on your server affect the SDK's behavior directly and take precedence over the matching init options:
Expand Down
2 changes: 1 addition & 1 deletion lib/countly-bulk-user.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ var cc = require("./countly-common");
* @param {boolean} [conf.require_consent=false] - Pass true if you are implementing GDPR compatible consent management. It would prevent running any functionality without proper consent
* @param {number} [conf.max_key_length=128] - maximum size of all string keys
* @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars)
* @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
Expand Down
8 changes: 4 additions & 4 deletions lib/countly-bulk.js
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ CountlyBulk.StorageTypes = cc.storageTypeEnums;
* @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request
* @param {number} [conf.max_key_length=128] - maximum size of all string keys
* @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars)
* @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
Expand Down Expand Up @@ -252,8 +252,8 @@ function CountlyBulk(conf) {
if (!event.count) {
event.count = 1;
}
event.key = cc.truncateSingleValue(event.key, self.maxKeyLength, "add_event", self.debug);
event.segmentation = cc.truncateObject(event.segmentation, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_event", self.debug);
event.key = cc.truncateSingleValue(event.key, conf.maxKeyLength, "add_event");
event.segmentation = cc.truncateObject(event.segmentation, conf.maxKeyLength, conf.maxValueSize, conf.maxSegmentationValues, "add_event");
var props = ["key", "count", "sum", "dur", "segmentation", "timestamp"];
var e = cc.makeJsonSafe(cc.getProperties(event, props), "add_event");
e.timestamp = e.timestamp || getMsTimestamp();
Expand Down Expand Up @@ -352,7 +352,7 @@ function CountlyBulk(conf) {
* var user = server.add_user({device_id:"my_device_id"});
* */
this.add_user = function(userConf) {
userConf = cc.truncateObject(userConf, self.maxKeyLength, self.maxValueSize, self.maxSegmentationValues, "add_user", self.debug);
userConf = cc.makeJsonSafe(userConf, "add_user") || {};
userConf.server = this;
return new BulkUser(userConf);
};
Expand Down
2 changes: 1 addition & 1 deletion lib/countly.js
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ Countly.Bulk = Bulk;
* @deprecated {number} [conf.max_logs=100] - maximum amount of breadcrumbs to store for crash logs
* @param {number} [conf.max_key_length=128] - maximum size of all string keys
* @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars)
* @param {number} [conf.max_segmentation_values=30] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_segmentation_values=100] - max amount of custom (dev provided) segmentation in one event
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
Expand Down
23 changes: 23 additions & 0 deletions test/helpers/scripts/bulk_cluster_limits.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
const cluster = require("cluster");
const CountlyBulk = require("../../../lib/countly-bulk");

const server = new CountlyBulk({
app_key: "YOUR_APP_KEY",
url: process.env.COUNTLY_URL,
interval: 200,
max_key_length: 4,
max_value_size: 5,
});

if (cluster.isMaster) {
server.start();
const worker = cluster.fork();
setTimeout(() => {
worker.kill();
process.exit(0);
}, Number(process.env.RUN_MS || 8000));
}
else {
server.add_event("cluster-limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value" } });
setInterval(() => { }, 1000);
}
77 changes: 77 additions & 0 deletions test/tests_bulk_fixes.js
Original file line number Diff line number Diff line change
Expand Up @@ -185,4 +185,81 @@ describe("Bulk fix tests", () => {
await server.close();
assert.ok(resent, "the bulk queue stopped after the cut-off response");
});

it("10- events reach the server cut to the internal limits set on the bulk instance", async() => {
const server = await startMockServer();
const bulk = new CountlyBulk({ app_key: appKey, url: server.url, interval: 100, max_key_length: 4, max_value_size: 5, max_segmentation_values: 2 });
bulk.add_event("limits-user", { key: "checkout_completed", segmentation: { category: "overlong-value", plan: "pro", coupon: "dropped" } });
bulk.start();
const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "limits-user"), 10000);
bulk.stop();
await server.close();
assert.ok(sent, "the event was not sent");
const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "limits-user")[0];
assert.equal(events.length, 1);
assert.equal(events[0].key, "chec");
assert.deepStrictEqual(events[0].segmentation, { cate: "overl", plan: "pro" });
});

it("11- events added to the bulk instance keep to the default internal limits", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1" });
const segmentation = {};
for (let i = 0; i < 101; i++) {
segmentation[`${i}-${"k".repeat(200)}`] = "v".repeat(300);
}
bulk.add_event("default-limits-user", { key: "e".repeat(200), segmentation: segmentation });
const event = bulk._getBulkEventQueue()["default-limits-user"][0];
assert.equal(event.key, "e".repeat(128));
const keys = Object.keys(event.segmentation);
assert.equal(keys.length, 100);
keys.forEach((key, i) => {
assert.equal(key, `${i}-${"k".repeat(200)}`.substring(0, 128));
assert.equal(event.segmentation[key], "v".repeat(256));
});
});

it("12- events of a bulk user keep to the internal limits set on the bulk instance", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5 });
bulk.add_user({ device_id: "limits-bulk-user" }).add_event({ key: "checkout_completed", segmentation: { category: "overlong-value" } });
const events = bulk._getBulkEventQueue()["limits-bulk-user"];
assert.equal(events.length, 1);
assert.equal(events[0].key, "chec");
assert.deepStrictEqual(events[0].segmentation, { cate: "overl" });
});

it("13- events that a cluster worker hands to the main process keep to the main process's internal limits", async() => {
const server = await startMockServer();
const run = runScript("bulk_cluster_limits.js", { COUNTLY_URL: server.url, RUN_MS: "8000" }, 30000);
const sent = await server.waitFor((reqs) => bulkInnerRequests(reqs).some((r) => r.device_id === "cluster-limits-user"), 20000);
const result = await run;
await server.close();
assert.ok(sent, `the worker's event was not sent: ${result.stderr}`);
const { events } = bulkInnerRequests(server.requests).filter((r) => r.device_id === "cluster-limits-user")[0];
assert.equal(events.length, 1);
assert.equal(events[0].key, "chec");
assert.deepStrictEqual(events[0].segmentation, { cate: "overl" });
});

it("14- a bulk user keeps its settings whole when the bulk instance has low internal limits", () => {
// the limits are shorter than the setting names and values, and allow one setting only
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 8, max_value_size: 5, max_segmentation_values: 1 });
const settings = { ip_address: "203.0.113.7", country_code: "TR", city: "Istanbul", require_consent: true };
const withoutConsent = bulk.add_user({ device_id: "settings-user-without-consent", ...settings });
withoutConsent.begin_session();
withoutConsent.add_event({ key: "purchase" });
const withConsent = bulk.add_user({ device_id: "settings-user-with-consent", ...settings });
withConsent.add_consent(["sessions", "location", "events"]);
withConsent.begin_session();
withConsent.add_event({ key: "purchase" });

const requests = bulk._getBulkRequestQueue();
assert.equal(requests.filter((r) => r.device_id === "settings-user-without-consent").length, 0, "a request of the user without consent was queued");
assert.equal(bulk._getBulkEventQueue()["settings-user-without-consent"], undefined, "an event of the user without consent was queued");
const session = requests.find((r) => r.device_id === "settings-user-with-consent" && r.begin_session === 1);
assert.ok(session, "the session of the user with consent was not queued");
assert.equal(session.ip_address, "203.0.113.7");
assert.equal(session.country_code, "TR");
assert.equal(session.city, "Istanbul");
assert.equal(bulk._getBulkEventQueue()["settings-user-with-consent"].length, 1);
});
});
Loading