Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
- Mitigated an issue where `add_event` and `end_event` threw an error when called without an event
- Mitigated an issue where events added to `CountlyBulk` ignored its `max_key_length`, `max_value_size` and `max_segmentation_values` limits
- Mitigated an issue where bulk users ignored the limits set for `CountlyBulk` and used the default limits instead
- Mitigated an issue where the custom metrics of `report_trace` ignored the `max_key_length`, `max_value_size` and `max_segmentation_values` limits
- Mitigated an issue where `report_app_start` sent no app start trace
- Mitigated an issue where crash reports of bulk users ignored the stack trace, breadcrumb and crash segmentation limits

## 26.8.0
- ! Minor breaking change ! SDK behavior settings are now enabled by default. Changes made in SDK Manager on your server affect the SDK's behavior directly and take precedence over the matching init options:
Expand Down
47 changes: 40 additions & 7 deletions lib/countly-bulk-user.js
Original file line number Diff line number Diff line change
Expand Up @@ -528,23 +528,23 @@ function CountlyBulkUser(conf) {
* @param {string} trace.name - url or view of the trace
* @param {number} trace.stz - start timestamp
* @param {number} trace.etz - end timestamp
* @param {Object} trace.app_metrics - key/value metrics like duration, to report with trace where value is number
* @param {Object} trace.apm_metrics - key/value metrics like duration, to report with trace where value is number. Custom metrics keep to the internal limits, the reserved metrics such as duration and response_time are sent whole
* @param {Object=} trace.apm_attr - object profiling attributes (not yet supported)
* @returns {module:lib/countly-bulk-user} instance
*/
this.report_trace = function(trace) {
if (this.check_consent("apm")) {
trace.name = cc.truncateSingleValue(trace.name, conf.maxKeyLength, "report_trace");
trace.app_metrics = cc.truncateObject(trace.app_metrics, conf.maxKeyLength, conf.maxValueSize, conf.maxSegmentationValues, "report_trace");
var props = ["type", "name", "stz", "etz", "apm_metrics", "apm_attr"];
for (var i = 0; i < props.length; i++) {
if (props[i] !== "apm_attr" && typeof trace[props[i]] === "undefined") {
cc.log(cc.logLevelEnums.ERROR, `CountlyBulkUser report_trace, APM trace must have a: [${props[i]}] param.`);
return;
return this;
}
}

var e = cc.getProperties(trace, props);
e.apm_metrics = cc.truncateApmMetrics(e.apm_metrics, conf.maxKeyLength, conf.maxValueSize, conf.maxSegmentationValues, "report_trace");
e.timestamp = trace.stz;
var date = new Date();
e.hour = date.getHours();
Expand Down Expand Up @@ -580,17 +580,18 @@ function CountlyBulkUser(conf) {
* @param {boolean=} crash._muted - true if volume is off, device is in muted state
* @param {boolean=} crash._background - true if app was in background when it crashed
* @param {string=} crash._name - identfiiable name of the crash if provided by OS/Platform, else will use first line of stack
* @param {string} crash._error - error stack, can provide multiple separated by blank new lines
* @param {string} crash._error - error stack, can provide multiple separated by blank new lines. Cut to the stack trace limits (max_stack_trace_lines_per_thread lines of max_stack_trace_line_length characters per thread, 50 threads, 20,000 characters), unless _native_cpp is true
* @param {boolean=} crash._native_cpp - true if _error holds a native crash dump, which is sent whole
* @param {boolean=} crash._nonfatal - true if handled exception, false or not provided if unhandled crash
* @param {string=} crash._logs - some additional logs provided, if any
* @param {string=} crash._logs - breadcrumbs joined with new lines. The newest max_breadcrumb_count are kept, each cut to max_value_size characters
* @param {number=} crash._run - running time since app start in seconds until crash
* @param {string=} crash._custom - custom key values to record with crash report, like versions of other libraries and frameworks used, etc.
* @param {Object=} crash._custom - custom key values to record with crash report, like versions of other libraries and frameworks used, etc. Keeps to the key, value and segmentation limits
* @param {number} timestamp - when crash happened
* @returns {module:lib/countly-bulk-user} instance
* */
this.report_crash = function(crash, timestamp) {
if (this.check_consent("crashes")) {
var query = prepareQuery({ crash: crash });
var query = prepareQuery({ crash: limitCrash(crash) });
if (timestamp) {
query.timestamp = timestamp;
}
Expand Down Expand Up @@ -797,6 +798,38 @@ function CountlyBulkUser(conf) {
}
return query;
}

/**
* Returns a copy of a crash report within the crash limits: the stack trace unless it is a native crash dump,
* the newest breadcrumbs and the crash segmentation
* @param {Object} crash - crash report as given to report_crash
* @returns {Object} the crash report within the limits
*/
function limitCrash(crash) {
var limited = cc.makeJsonSafe(crash, "report_crash");
if (!limited || typeof limited !== "object") {
return limited;
}
if (typeof limited._error === "string" && !limited._native_cpp) {
var error = cc.truncateStackTrace(limited._error, conf.maxStackTraceLinesPerThread, conf.maxStackTraceLineLength);
if (error !== limited._error) {
cc.log(cc.logLevelEnums.WARNING, `CountlyBulkUser report_crash, The error text is longer than [${conf.maxStackTraceLinesPerThread}] lines of [${conf.maxStackTraceLineLength}] characters per thread, it is cut.`);
limited._error = error;
}
}
if (typeof limited._logs === "string") {
var breadcrumbs = limited._logs.split("\n");
if (breadcrumbs.length > conf.maxBreadcrumbCount) {
breadcrumbs = breadcrumbs.slice(breadcrumbs.length - conf.maxBreadcrumbCount);
cc.log(cc.logLevelEnums.DEBUG, "CountlyBulkUser report_crash, Breadcrumbs overflowed. Erasing the oldest.");
}
limited._logs = breadcrumbs.map((record) => cc.truncateSingleValue(record, conf.maxValueSize, "report_crash")).join("\n");
}
if (limited._custom && typeof limited._custom === "object") {
limited._custom = cc.truncateObject(limited._custom, conf.maxKeyLength, conf.maxValueSize, conf.maxSegmentationValues, "report_crash");
}
return limited;
}
}

module.exports = CountlyBulkUser;
58 changes: 58 additions & 0 deletions lib/countly-common.js
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,64 @@ var cc = {
return this.makeJsonSafe(ob, errorLog);
},

// APM metric keys the server reads itself, which truncateApmMetrics keeps whole
apmReservedMetrics: ["response_time", "response_payload_size", "response_code", "request_payload_size", "duration", "slow_rendering_frames", "frozen_frames"],

/**
* Applies the internal limits to the custom metrics of an APM trace. The reserved metrics are kept whole and do
* not count towards the segmentation limit, and a custom metric cut to a reserved key does not replace it.
* @param {Object} metrics - apm_metrics of a trace
* @param {Number} keyLimit - limit for key length
* @param {Number} valueLimit - limit for value length
* @param {Number} segmentLimit - limit for the number of custom metrics
* @param {string} errorLog - prefix for error log
* @param {String=} area - health check feature area, see truncateObject
* @returns {Object} the metrics within the limits
*/
truncateApmMetrics: function truncateApmMetrics(metrics, keyLimit, valueLimit, segmentLimit, errorLog, area) {
if (!metrics || typeof metrics !== "object") {
return metrics;
}
var result = {};
var custom = {};
Object.keys(metrics).forEach((key) => {
if (this.apmReservedMetrics.indexOf(key) !== -1) {
result[key] = metrics[key];
}
else {
custom[key] = metrics[key];
}
});
var limited = this.truncateObject(custom, keyLimit, valueLimit, segmentLimit, errorLog, area);
Object.keys(limited).forEach((key) => {
if (typeof result[key] === "undefined") {
result[key] = limited[key];
}
});
return this.makeJsonSafe(result, errorLog);
},

/**
* Cuts a stack trace to the stack trace limits. Threads are separated by blank lines: at most 50 threads are kept,
* each with at most maxLines lines of at most maxLineLength characters, and the whole text is cut to 20,000
* characters. A stack trace within the limits is returned as it was.
* @param {String} stack - stack trace text
* @param {Number} maxLines - limit for the lines of one thread
* @param {Number} maxLineLength - limit for the length of one line
* @returns {String} the stack trace within the limits
*/
truncateStackTrace: function truncateStackTrace(stack, maxLines, maxLineLength) {
var maxThreads = 50;
var maxLength = 20000;
var parts = stack.split(/(\n[ \t\r]*\n)/);
var result = "";
for (var thread = 0; thread < maxThreads && thread * 2 < parts.length; thread++) {
var lines = parts[thread * 2].split("\n").slice(0, maxLines).map((line) => line.substring(0, maxLineLength));
result += (thread > 0 ? parts[(thread * 2) - 1] : "") + lines.join("\n");
}
return result.substring(0, maxLength);
},

/**
* Returns a copy of a value that JSON.stringify can always serialize, with the same JSON as the
* original wherever the original could be serialized. BigInt numbers become strings, so database
Expand Down
12 changes: 6 additions & 6 deletions lib/countly.js
Original file line number Diff line number Diff line change
Expand Up @@ -1874,13 +1874,12 @@ Countly.Bulk = Bulk;
* @param {string} trace.name - url or view of the trace
* @param {number} trace.stz - start timestamp
* @param {number} trace.etz - end timestamp
* @param {Object} trace.app_metrics - key/value metrics like duration, to report with trace where value is number
* @param {Object} trace.apm_metrics - key/value metrics like duration, to report with trace where value is number. Custom metrics keep to the internal limits, the reserved metrics such as duration and response_time are sent whole
* @param {Object=} trace.apm_attr - object profiling attributes (not yet supported)
*/
Countly.report_trace = function(trace) {
if (Countly.check_consent("apm")) {
trace.name = cc.truncateSingleValue(trace.name, Countly.maxKeyLength, "report_trace", "w201");
trace.app_metrics = cc.truncateObject(trace.app_metrics, Countly.maxKeyLength, Countly.maxValueSize, Countly.maxSegmentationValues, "report_trace", "");
var props = ["type", "name", "stz", "etz", "apm_metrics", "apm_attr"];
for (var i = 0; i < props.length; i++) {
if (props[i] !== "apm_attr" && typeof trace[props[i]] === "undefined") {
Expand All @@ -1891,6 +1890,7 @@ Countly.Bulk = Bulk;
}

var e = cc.getProperties(trace, props);
e.apm_metrics = cc.truncateApmMetrics(e.apm_metrics, Countly.maxKeyLength, Countly.maxValueSize, Countly.maxSegmentationValues, "report_trace", "");
e.timestamp = trace.stz;
var date = new Date();
e.hour = date.getHours();
Expand All @@ -1903,21 +1903,21 @@ Countly.Bulk = Bulk;
};

/**
* Report time passed since app start trace
* Report the time from the start of the process until now as the "app_start" trace
*/
Countly.report_app_start = function() {
cc.log(cc.logLevelEnums.INFO, "report_app_start, Reporting app start.");

// do on next tick to allow synchronous code to load
process.nextTick(() => {
var start = Math.floor(process.uptime() * 1000);
var end = Date.now();
var start = end - Math.floor(process.uptime() * 1000);
Countly.report_trace({
type: "device",
name: process.title || process.argv.join(" "),
name: "app_start",
stz: start,
etz: end,
app_metrics: {
apm_metrics: {
duration: end - start,
},
});
Expand Down
81 changes: 81 additions & 0 deletions test/tests_bulk_fixes.js
Original file line number Diff line number Diff line change
Expand Up @@ -285,4 +285,85 @@ describe("Bulk fix tests", () => {
const request = bulk._getBulkRequestQueue().find((r) => r.device_id === "own-limits-user");
assert.deepStrictEqual(request.user_details, { name: "Long nam", custom: {} });
});

it("17- custom APM metrics of a bulk user keep to the internal limits, and the reserved metrics stay whole", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 8, max_segmentation_values: 2 });
const user = bulk.add_user({ device_id: "apm-limits-user" });
user.report_trace({
type: "network",
name: "/checkout",
stz: 1700000000000,
etz: 1700000001330,
apm_metrics: { response_time: 1330, response_payload_size: 120, response_code: 200, request_payload_size: 70, cpu_usage_percent: 45, memory_used_mb: 512, disk_reads_count: 7 },
});
assert.strictEqual(user.report_trace({ type: "device", name: "no_metrics", stz: 1700000000000, etz: 1700000001000 }), user);
const traces = bulk._getBulkRequestQueue().filter((r) => r.device_id === "apm-limits-user" && r.apm).map((r) => JSON.parse(r.apm));
assert.equal(traces.length, 1, "a trace without metrics was queued");
assert.equal(traces[0].name, "/checkou");
assert.deepStrictEqual(traces[0].apm_metrics, { response_time: 1330, response_payload_size: 120, response_code: 200, request_payload_size: 70, cpu_usag: 45, memory_u: 512 });
});

it("18- crash reports of a bulk user keep to the stack trace, breadcrumb and crash segmentation limits", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_key_length: 4, max_value_size: 5, max_segmentation_values: 1, max_breadcrumb_count: 2, max_stack_trace_lines_per_thread: 2, max_stack_trace_line_length: 10 });
const error = "Error: something failed\n at first (a.js:1:1)\n at second (a.js:2:2)\n\nThread 2 started\n at third (b.js:3:3)\n at fourth (b.js:4:4)";
const logs = "first step\nsecond step\nthird step";
const crash = { _os: "Linux", _os_version: "6", _app_version: "1.0", _error: error, _logs: logs, _custom: { library: "version-1.2.3", other: "dropped" }, _nonfatal: true };
bulk.add_user({ device_id: "crash-limits-user" }).report_crash(crash, 1700000000);
const request = bulk._getBulkRequestQueue().find((r) => r.device_id === "crash-limits-user");
assert.equal(request.timestamp, 1700000000);
assert.deepStrictEqual(request.crash, {
_os: "Linux",
_os_version: "6",
_app_version: "1.0",
_error: "Error: som\n at fir\n\nThread 2 s\n at thi",
_logs: "secon\nthird",
_custom: { libr: "versi" },
_nonfatal: true,
});
assert.equal(crash._error, error, "the reported crash object was changed");
assert.equal(crash._logs, logs, "the reported crash object was changed");
});

it("19- a native crash dump of a bulk user is sent whole, while the same text in a JavaScript crash is cut", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1", max_stack_trace_lines_per_thread: 2, max_stack_trace_line_length: 10 });
const dump = "A".repeat(30000);
const user = bulk.add_user({ device_id: "native-crash-user" });
user.report_crash({ _os: "Android", _os_version: "14", _app_version: "1.0", _error: dump, _native_cpp: true });
user.report_crash({ _os: "Android", _os_version: "14", _app_version: "1.0", _error: dump });
const errors = bulk._getBulkRequestQueue().filter((r) => r.device_id === "native-crash-user").map((r) => r.crash._error);
assert.equal(errors.length, 2);
assert.equal(errors[0], dump);
assert.equal(errors[1], "AAAAAAAAAA");
});

it("20- without set limits, a bulk crash keeps 50 threads of 30 lines of 200 characters, 20,000 characters in all, and 100 breadcrumbs", () => {
const bulk = new CountlyBulk({ app_key: appKey, url: "http://127.0.0.1:1" });
const user = bulk.add_user({ device_id: "default-crash-user" });
const threads = [];
for (let i = 0; i < 60; i++) {
threads.push(`Thread ${i}`);
}
const lines = [];
for (let i = 0; i < 40; i++) {
lines.push(`line ${i} ${"x".repeat(250)}`);
}
const fullThreads = [];
for (let i = 0; i < 5; i++) {
fullThreads.push(new Array(30).fill("y".repeat(200)).join("\n"));
}
const breadcrumbs = [];
for (let i = 0; i < 101; i++) {
breadcrumbs.push(`crumb ${i} ${"z".repeat(300)}`);
}
const base = { _os: "Linux", _os_version: "6", _app_version: "1.0" };
user.report_crash({ ...base, _error: threads.join("\n\n") });
user.report_crash({ ...base, _error: lines.join("\n") });
user.report_crash({ ...base, _error: fullThreads.join("\n\n"), _logs: breadcrumbs.join("\n") });
const crashes = bulk._getBulkRequestQueue().filter((r) => r.device_id === "default-crash-user").map((r) => r.crash);
assert.equal(crashes.length, 3);
assert.equal(crashes[0]._error, threads.slice(0, 50).join("\n\n"));
assert.equal(crashes[1]._error, lines.slice(0, 30).map((line) => line.substring(0, 200)).join("\n"));
assert.equal(crashes[2]._error, fullThreads.join("\n\n").substring(0, 20000));
assert.equal(crashes[2]._logs, breadcrumbs.slice(1).map((crumb) => crumb.substring(0, 256)).join("\n"));
});
});
23 changes: 23 additions & 0 deletions test/tests_internal_limits.js
Original file line number Diff line number Diff line change
Expand Up @@ -252,4 +252,27 @@ describe("Testing internal limits", () => {
done();
}, hp.sWait);
});
it("6. Check APM trace truncation", (done) => {
// init Countly
initLimitsMain();
// custom metrics keep to the limits, the reserved network metrics stay whole
Countly.report_trace({
type: "network",
name: "/checkout/payment",
stz: 1700000000000,
etz: 1700000001330,
apm_metrics: { response_time: 1330, response_payload_size: 120, response_code: 200, request_payload_size: 70, cpu_usage_percent: 45, memory_used_mb: 512, disk_reads_count: 7, net_errors_count: 2 },
});
// a trace without metrics is still refused
Countly.report_trace({ type: "device", name: "no_metrics", stz: 1700000000000, etz: 1700000001000 });
// test
setTimeout(() => {
var rq = hp.readRequestQueue();
assert.equal(rq.length, 1);
var apm = JSON.parse(rq[0].apm);
assert.equal(apm.name, "/checkou");
assert.deepStrictEqual(apm.apm_metrics, { response_time: 1330, response_payload_size: 120, response_code: 200, request_payload_size: 70, cpu_usag: 45, memory_u: 512, disk_rea: 7 });
done();
}, hp.sWait);
});
});
Loading
Loading