diff --git a/.env.sample b/.env.sample index f2df9be2..a5ecb4dc 100644 --- a/.env.sample +++ b/.env.sample @@ -55,5 +55,8 @@ HAWK_CATCHER_TOKEN= ## If true, Grouper worker will send messages about new events to Notifier worker IS_NOTIFIER_WORKER_ENABLED=false +## Comma-separated workspace ids that should use dailyEvents counters in Limiter quota checks. Use * for all workspaces. +LIMITER_DAILY_EVENTS_COUNTER_WORKSPACE_IDS= + ## Url for telegram notifications about workspace blocks and unblocks TELEGRAM_LIMITER_CHAT_URL= diff --git a/lib/db/controller.ts b/lib/db/controller.ts index 07de49ac..9cd24e18 100644 --- a/lib/db/controller.ts +++ b/lib/db/controller.ts @@ -87,7 +87,8 @@ export class DatabaseController { useNewUrlParser: true, useUnifiedTopology: true, serverSelectionTimeoutMS: SERVER_SELECTION_TIMEOUT_MS, - ...(this.appName ? { appName: this.appName } : {}), + /** driver 3.x only recognizes the lowercase `appname` option key */ + ...(this.appName ? { appname: this.appName } : {}), }); this.db = this.connection.db(); diff --git a/lib/utils/sanitizer.ts b/lib/utils/sanitizer.ts new file mode 100644 index 00000000..dab7686d --- /dev/null +++ b/lib/utils/sanitizer.ts @@ -0,0 +1,129 @@ +import { rightTrim } from './string'; + +/** + * Maximum string length before appending ellipsis + */ +const MAX_STRING_LENGTH = 200; + +/** + * Maximum number of object keys to keep, the rest are reported via META_FIELD + */ +const MAX_OBJECT_KEYS_COUNT = 20; + +/** + * Key added to an object to report how many of its keys were skipped + */ +const META_FIELD = '__meta'; + +/** + * Maximum depth of sanitized objects + */ +const MAX_DEPTH = 5; + +/** + * Maximum length of sanitized arrays + */ +const MAX_ARRAY_LENGTH = 10; + +/** + * Checks that the value is a plain object + * + * @param target - value to check + */ +function isPlainObject(target: unknown): target is Record { + return Object.prototype.toString.call(target) === '[object Object]'; +} + +/** + * Values that can be tracked by WeakSet to detect circular references + */ +type ObjectLike = Record | unknown[]; + +/** + * Prepares event data for storing: trims long strings, slices long arrays, + * cuts off extra object keys and replaces too deep objects and circular + * references with placeholders. + */ +export class Sanitizer { + /** + * Apply sanitizing for array/object/primitives + * + * @param data - any value to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + public static sanitize(data: unknown, depth = 0, seen = new WeakSet()): unknown { + if (data !== null && typeof data === 'object') { + if (seen.has(data as ObjectLike)) { + return ''; + } + seen.add(data as ObjectLike); + } + + if (Array.isArray(data)) { + return Sanitizer.sanitizeArray(data, depth + 1, seen); + } + + if (isPlainObject(data)) { + return Sanitizer.sanitizeObject(data, depth + 1, seen); + } + + if (typeof data === 'string') { + return rightTrim(data, MAX_STRING_LENGTH); + } + + return data; + } + + /** + * Slices array to the maximum length and sanitizes each element + * + * @param arr - array to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + private static sanitizeArray(arr: unknown[], depth: number, seen: WeakSet): unknown[] { + const length = arr.length; + + if (length > MAX_ARRAY_LENGTH) { + arr = arr.slice(0, MAX_ARRAY_LENGTH); + arr.push(`<${length - MAX_ARRAY_LENGTH} more items...>`); + } + + return arr.map((item) => { + return Sanitizer.sanitize(item, depth, seen); + }); + } + + /** + * Sanitizes object values recursively + * + * @param data - object to sanitize + * @param depth - current depth of recursion + * @param seen - already visited objects + */ + private static sanitizeObject( + data: Record, + depth: number, + seen: WeakSet + ): Record | '' { + if (depth > MAX_DEPTH) { + return ''; + } + + const keys = Object.keys(data); + const result: Record = {}; + + for (const key of keys.slice(0, MAX_OBJECT_KEYS_COUNT)) { + result[key] = Sanitizer.sanitize(data[key], depth, seen); + } + + const skippedKeysCount = keys.length - MAX_OBJECT_KEYS_COUNT; + + if (skippedKeysCount > 0) { + result[META_FIELD] = `${skippedKeysCount} more key(s) skipped`; + } + + return result; + } +} diff --git a/workers/grouper/src/data-filter.ts b/workers/grouper/src/data-filter.ts index 2345b8e5..bd89d175 100644 --- a/workers/grouper/src/data-filter.ts +++ b/workers/grouper/src/data-filter.ts @@ -1,11 +1,18 @@ import type { EventAddons, EventData } from '@hawk.so/types'; import { unsafeFields } from '../../../lib/utils/unsafeFields'; +import { rightTrim } from '../../../lib/utils/string'; +import { Sanitizer } from '../../../lib/utils/sanitizer'; /** * Maximum depth for object traversal to prevent excessive memory allocations */ const MAX_TRAVERSAL_DEPTH = 20; +/** + * Maximum length for event title before appending ellipsis + */ +const MAX_TITLE_LENGTH = 400; + /** * Recursively iterate through object and call function on each key * @@ -135,6 +142,9 @@ export default class DataFilter { * @param event - event to process */ public processEvent(event: EventData): void { + this.trimEventTitle(event); + this.sanitizeEvent(event); + unsafeFields.forEach(field => { if (event[field]) { this.processField(event[field]); @@ -142,6 +152,48 @@ export default class DataFilter { }); } + /** + * Trim event title to the maximum allowed length. + * It mutates the original object. + * + * @param event - event to process + */ + public trimEventTitle(event: EventData): void { + if (typeof event.title === 'string') { + event.title = rightTrim(event.title, MAX_TITLE_LENGTH); + } + } + + /** + * Sanitize event fields that can contain long strings, deep objects or long arrays. + * It mutates the original object. + * + * @param event - event to process + */ + public sanitizeEvent(event: EventData): void { + unsafeFields.forEach(field => { + if (event[field] !== undefined) { + (event as unknown as Record)[field] = Sanitizer.sanitize(event[field]); + } + }); + + event.backtrace?.forEach(frame => { + if (frame.arguments !== undefined) { + frame.arguments = Sanitizer.sanitize(frame.arguments) as string[]; + } + }); + + event.breadcrumbs?.forEach(breadcrumb => { + if (typeof breadcrumb.message === 'string') { + breadcrumb.message = Sanitizer.sanitize(breadcrumb.message) as string; + } + + if (breadcrumb.data !== undefined) { + breadcrumb.data = Sanitizer.sanitize(breadcrumb.data) as typeof breadcrumb.data; + } + }); + } + /** * Recursively iterates object and applies filtering to its entries * diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 8203e8d3..46f77c98 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -59,7 +59,7 @@ const DB_DUPLICATE_KEY_ERROR = '11000'; const DAILY_METRICS_RETENTION_DAYS = 90; /** - * Maximum length for backtrace code line or title + * Maximum length for backtrace code line */ const MAX_CODE_LINE_LENGTH = 140; @@ -198,8 +198,6 @@ export default class GrouperWorker extends Worker { this.grouperMetrics.observePayloadSize(taskPayloadSize); this.memoryMonitor.logBeforeHandle(memoryBeforeHandle, handledTasksCount, taskPayloadSize, task.projectId); - this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`); - // FIX RELEASE TYPE // TODO: REMOVE AFTER 01.01.2026, after the most of the users update to new js catcher if (task.payload && task.payload.release !== undefined) { @@ -209,6 +207,11 @@ export default class GrouperWorker extends Worker { }; } + /** + * Filter event data before hashing so hash and stored event stay consistent. + */ + this.dataFilter.processEvent(task.payload); + let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task)); let existedEvent: GroupedEventDBScheme; let repetitionId = null; @@ -219,11 +222,6 @@ export default class GrouperWorker extends Worker { * Trim source code lines to prevent memory leaks */ this.trimSourceCodeLines(task.payload); - - /** - * Filter sensitive information - */ - this.dataFilter.processEvent(task.payload); }); /** diff --git a/workers/grouper/tests/data-filter.test.ts b/workers/grouper/tests/data-filter.test.ts index 2f00dd68..e4ec8164 100644 --- a/workers/grouper/tests/data-filter.test.ts +++ b/workers/grouper/tests/data-filter.test.ts @@ -14,16 +14,24 @@ jest.mock('amqplib'); * @param [options.context] - generated event context * @param [options.addons] - generated event addons */ -function generateEvent({ context, addons }: {context?: Json, addons?: EventAddons}): EventData { +function generateEvent({ context, addons, backtrace, breadcrumbs }: { + context?: Json, + addons?: EventAddons, + backtrace?: EventData['backtrace'], + breadcrumbs?: EventData['breadcrumbs'], +}): EventData { return { title: 'Event with sensitive data', - backtrace: [], + backtrace: backtrace ?? [], ...(context && { context, }), ...(addons && { addons, }), + ...(breadcrumbs && { + breadcrumbs, + }), }; } @@ -161,30 +169,52 @@ describe('GrouperWorker', () => { }); test('should filter additional sensitive keys (authorization, token, payment, dsn, ssn, etc.) in context', async () => { + /** + * Split the mock into two groups to keep objects under the sanitizer keys limit + */ + const entries = Object.entries(additionalSensitiveDataMock); + const half = Math.ceil(entries.length / 2); const event = generateEvent({ - context: additionalSensitiveDataMock, + context: { + group1: Object.fromEntries(entries.slice(0, half)), + group2: Object.fromEntries(entries.slice(half)), + }, }); dataFilter.processEvent(event); - Object.keys(additionalSensitiveDataMock).forEach((key) => { - expect(event.context[key]).toBe('[filtered]'); + entries.slice(0, half).forEach(([ key ]) => { + expect(event.context['group1'][key]).toBe('[filtered]'); + }); + entries.slice(half).forEach(([ key ]) => { + expect(event.context['group2'][key]).toBe('[filtered]'); }); }); test('should filter additional sensitive keys in addons', async () => { + /** + * Split the mock into two groups to keep objects under the sanitizer keys limit + */ + const entries = Object.entries(additionalSensitiveDataMock); + const half = Math.ceil(entries.length / 2); const event = generateEvent({ addons: { vue: { - props: additionalSensitiveDataMock, + props: { + group1: Object.fromEntries(entries.slice(0, half)), + group2: Object.fromEntries(entries.slice(half)), + }, }, }, }); dataFilter.processEvent(event); - Object.keys(additionalSensitiveDataMock).forEach((key) => { - expect(event.addons['vue']['props'][key]).toBe('[filtered]'); + entries.slice(0, half).forEach(([ key ]) => { + expect(event.addons['vue']['props']['group1'][key]).toBe('[filtered]'); + }); + entries.slice(half).forEach(([ key ]) => { + expect(event.addons['vue']['props']['group2'][key]).toBe('[filtered]'); }); }); @@ -328,9 +358,9 @@ describe('GrouperWorker', () => { expect(event.context['auth']).toBe('[filtered]'); }); - test('should handle deeply nested objects (>20 levels) without excessive memory allocations', () => { - // Create an object nested deeper than the cap (>20 levels) - let deeplyNested: any = { value: 'leaf', secret: 'should-be-filtered' }; + test('should replace too deep objects with a placeholder and keep filtering reachable levels', () => { + // Create an object nested deeper than the sanitizer depth cap + let deeplyNested: any = { value: 'leaf', secret: 'should-be-cut-off' }; for (let i = 0; i < 25; i++) { deeplyNested = { [`level${i}`]: deeplyNested, password: `sensitive${i}` }; @@ -343,23 +373,148 @@ describe('GrouperWorker', () => { // This should not throw or cause memory issues dataFilter.processEvent(event); - // Verify that filtering still works at various depths + // Filtering still works on the levels kept by the sanitizer expect(event.context['password']).toBe('[filtered]'); - // Navigate to a mid-level and check filtering - let current = event.context['level24'] as any; - for (let i = 24; i > 15; i--) { - expect(current['password']).toBe('[filtered]'); - current = current[`level${i - 1}`]; + const deepestKeptLevel = event.context['level24']['level23']['level22']['level21']; + + expect(deepestKeptLevel['password']).toBe('[filtered]'); + + // Everything deeper is replaced with a placeholder + expect(deepestKeptLevel['level20']).toBe(''); + }); + }); + + describe('Sanitizer', () => { + test('should trim long strings in context', () => { + const longString = 'a'.repeat(5000); + const event = generateEvent({ + context: { + longValue: longString, + }, + }); + + dataFilter.processEvent(event); + + expect(event.context['longValue']).toBe('a'.repeat(200) + '…'); + }); + + test('should cut off extra keys of nested objects with too many keys', () => { + const bigObject: Record = {}; + + for (let i = 0; i < 25; i++) { + bigObject[`key${i}`] = i; } - // At the leaf level, the secret should still be filtered - // (though path tracking may be capped, filtering should still work) - let leaf = event.context; - for (let i = 24; i >= 0; i--) { - leaf = leaf[`level${i}`] as any; + const event = generateEvent({ + context: { + bigObject, + }, + }); + + dataFilter.processEvent(event); + + expect(Object.keys(event.context['bigObject'])).toHaveLength(21); + expect(event.context['bigObject']['__meta']).toBe('5 more key(s) skipped'); + }); + + test('should slice long arrays and add a placeholder', () => { + const event = generateEvent({ + context: { + longArray: new Array(15).fill('item') as unknown as Json, + }, + }); + + dataFilter.processEvent(event); + + const sanitizedArray = event.context['longArray']; + + expect(sanitizedArray).toHaveLength(11); // 10 items + placeholder + expect(sanitizedArray[10]).toBe('<5 more items...>'); + }); + + test('should sanitize addons', () => { + const event = generateEvent({ + addons: { + vue: { + props: { + longValue: 'b'.repeat(5000), + }, + }, + }, + }); + + dataFilter.processEvent(event); + + expect(event.addons['vue']['props']['longValue']).toBe('b'.repeat(200) + '…'); + }); + + test('should trim long backtrace frame arguments', () => { + const event = generateEvent({ + backtrace: [ { + file: 'index.js', + line: 1, + arguments: [ 'c'.repeat(5000) ], + } ], + }); + + dataFilter.processEvent(event); + + expect(event.backtrace[0].arguments[0]).toBe('c'.repeat(200) + '…'); + }); + + test('should sanitize breadcrumbs message and data', () => { + const event = generateEvent({ + breadcrumbs: [ { + timestamp: 1701867896789, + message: 'd'.repeat(5000), + data: { + longValue: 'e'.repeat(5000), + }, + } ], + }); + + dataFilter.processEvent(event); + + expect(event.breadcrumbs[0].message).toBe('d'.repeat(200) + '…'); + expect(event.breadcrumbs[0].data['longValue']).toBe('e'.repeat(200) + '…'); + }); + + test('should keep the first keys of a big object and report the skipped ones', () => { + const bigContext: Record = {}; + + for (let i = 0; i < 25; i++) { + bigContext[`key${i}`] = i; } - expect(leaf['secret']).toBe('[filtered]'); + + const event = generateEvent({ + context: bigContext, + }); + + dataFilter.processEvent(event); + + expect(Object.keys(event.context)).toHaveLength(21); + expect(event.context['key0']).toBe(0); + expect(event.context['key19']).toBe(19); + expect(event.context['key20']).toBeUndefined(); + expect(event.context['__meta']).toBe('5 more key(s) skipped'); + }); + + test('should replace circular references with a placeholder', () => { + const circular: Record = { + name: 'circular', + }; + + circular.self = circular; + + const event = generateEvent({ + context: circular as Json, + }); + + dataFilter.processEvent(event); + + expect(event.context['name']).toBe('circular'); + expect(event.context['self']).toBe(''); }); }); }); diff --git a/workers/grouper/tests/index.test.ts b/workers/grouper/tests/index.test.ts index c75351a9..e4f2b3bd 100644 --- a/workers/grouper/tests/index.test.ts +++ b/workers/grouper/tests/index.test.ts @@ -173,6 +173,20 @@ describe('GrouperWorker', () => { expect(await eventsCollection.find().count()).toBe(1); }); + test('Should trim event title to the maximum length before saving', async () => { + const longTitle = 'A'.repeat(5000); + + await worker.handle(generateTask({ title: longTitle })); + + const savedEvent = await eventsCollection.findOne({}); + + /** + * 400 chars + ellipsis + */ + expect(savedEvent.payload.title.length).toBe(401); + expect(savedEvent.payload.title.endsWith('…')).toBe(true); + }); + test('Should increment total events count on each processing', async () => { await worker.handle(generateTask()); await worker.handle(generateTask()); @@ -323,6 +337,9 @@ describe('GrouperWorker', () => { expect(typeof savedEvent.payload.context).toBe('string'); }); + /** + * Catchers never send a string context per spec — this covers the defensive fallback. + */ test('Should save event even if its context is type of string', async () => { const task = generateTask(); diff --git a/workers/limiter/src/dbHelper.ts b/workers/limiter/src/dbHelper.ts index 3af60b34..984dd335 100644 --- a/workers/limiter/src/dbHelper.ts +++ b/workers/limiter/src/dbHelper.ts @@ -3,6 +3,10 @@ import { PlanDBScheme, ProjectDBScheme, WorkspaceDBScheme } from '@hawk.so/types import { WorkspaceWithTariffPlan } from '../types'; import HawkCatcher from '@hawk.so/nodejs'; import { CriticalError, NonCriticalError } from '../../../lib/workerErrors'; +import { MS_IN_SEC } from '../../../lib/utils/consts'; +import TimeMs from '../../../lib/utils/time'; + +const SEC_IN_DAY = TimeMs.DAY / TimeMs.SECOND; const WORKSPACE_PROJECTION = { _id: 1, @@ -145,19 +149,13 @@ export class DbHelper { since: number ): Promise { try { - const repetitionsCollection = this.eventsDbConnection.collection('repetitions:' + project._id.toString()); - const eventsCollection = this.eventsDbConnection.collection('events:' + project._id.toString()); - const query = { timestamp: { $gt: since, }, }; - const repetitionsCount = await repetitionsCollection.countDocuments(query); - const originalEventCount = await eventsCollection.countDocuments(query); - - return repetitionsCount + originalEventCount; + return await this.getRawEventsCountByProject(project, query); } catch (e) { HawkCatcher.send(e); throw new CriticalError(e); @@ -179,6 +177,75 @@ export class DbHelper { .then(sum); } + /** + * Returns total event counts for last billing period using dailyEvents counters. + * + * Full days are summed from dailyEvents per-day counters (grouper + * increments `count` for originals and repetitions alike); only the + * partial day containing `since` is counted from the raw collections, + * since dailyEvents buckets have day granularity and lastChargeDate does not. + * + * @param project - project to check + * @param since - timestamp of the time from which we count the events + */ + public async getEventsCountByProjectUsingDailyEvents( + project: ProjectDBScheme, + since: number + ): Promise { + try { + const projectId = project._id.toString(); + const dailyEventsCollection = this.eventsDbConnection.collection('dailyEvents:' + projectId); + const firstFullDayTimestamp = this.getFirstFullDailyEventsTimestamp(since); + + const boundaryDayQuery = { + timestamp: { + $gt: since, + $lt: firstFullDayTimestamp, + }, + }; + + const [boundaryDayCount, dailyCounters] = await Promise.all([ + since < firstFullDayTimestamp + ? this.getRawEventsCountByProject(project, boundaryDayQuery) + : 0, + dailyEventsCollection + .aggregate<{ count: number }>([ + { $match: { groupingTimestamp: { $gte: firstFullDayTimestamp } } }, + { + $group: { + _id: null, + count: { $sum: '$count' }, + }, + }, + ]) + .toArray(), + ]); + + const fullDaysCount = dailyCounters.length > 0 ? dailyCounters[0].count : 0; + + return boundaryDayCount + fullDaysCount; + } catch (e) { + HawkCatcher.send(e); + throw new CriticalError(e); + } + } + + /** + * Calculates total events count for all provided projects since the specific date + * using dailyEvents counters for full days. + * + * @param projects - projects to calculate for + * @param since - timestamp of the time from which we count the events + */ + public async getEventsCountByProjectsUsingDailyEvents(projects: ProjectDBScheme[], since: number): Promise { + const sum = (array: number[]): number => array.reduce((acc, val) => acc + val, 0); + + return Promise.all(projects.map( + project => this.getEventsCountByProjectUsingDailyEvents(project, since) + )) + .then(sum); + } + /** * Returns all projects from Database or projects of the specified workspace * @@ -197,6 +264,52 @@ export class DbHelper { return this.projectsCollection.find(query).toArray(); } + /** + * UTC midnight right after the given timestamp. Mirrors grouper's + * getMidnightByEventTimestamp, which fills the dailyEvents buckets. + * + * @param timestamp - unix timestamp in seconds + */ + private getNextUtcMidnight(timestamp: number): number { + const date = new Date(timestamp * MS_IN_SEC); + + date.setUTCDate(date.getUTCDate() + 1); + date.setUTCHours(0, 0, 0, 0); + + return date.getTime() / MS_IN_SEC; + } + + /** + * Returns first dailyEvents bucket that can be safely used without counting + * events before the requested timestamp. + * + * @param timestamp - unix timestamp in seconds + */ + private getFirstFullDailyEventsTimestamp(timestamp: number): number { + const midnight = timestamp - (timestamp % SEC_IN_DAY); + + return timestamp === midnight ? timestamp : this.getNextUtcMidnight(timestamp); + } + + /** + * Counts raw original events and repetitions for the passed query. + * + * @param project - project to check + * @param query - MongoDB timestamp query + */ + private async getRawEventsCountByProject(project: ProjectDBScheme, query: Record): Promise { + const projectId = project._id.toString(); + const repetitionsCollection = this.eventsDbConnection.collection('repetitions:' + projectId); + const eventsCollection = this.eventsDbConnection.collection('events:' + projectId); + + const [repetitionsCount, originalEventCount] = await Promise.all([ + repetitionsCollection.countDocuments(query), + eventsCollection.countDocuments(query), + ]); + + return repetitionsCount + originalEventCount; + } + /** * Returns plan from cache, refetches once on miss * diff --git a/workers/limiter/src/index.ts b/workers/limiter/src/index.ts index 6ed21cfe..cc38d768 100644 --- a/workers/limiter/src/index.ts +++ b/workers/limiter/src/index.ts @@ -266,7 +266,7 @@ export default class LimiterWorker extends Worker { const since = Math.floor(new Date(workspace.lastChargeDate).getTime() / MS_IN_SEC); - const workspaceEventsCount = await this.dbHelper.getEventsCountByProjects(projects, since); + const workspaceEventsCount = await this.getWorkspaceEventsCount(workspace, projects, since); this.logger.info(`workspace ${workspace._id} events count since last charge date: ${workspaceEventsCount}`); @@ -328,6 +328,69 @@ export default class LimiterWorker extends Worker { }; } + /** + * Returns workspace events count using the default raw counter or the + * dailyEvents-based counter when it is explicitly enabled for the workspace. + * + * For enabled workspaces both counters are computed and their results with + * timings are reported to Telegram to compare the algorithms during the + * testing period. The old counter is used as a fallback if the new one fails. + * + * @param workspace - workspace to count events for + * @param projects - workspace projects + * @param since - timestamp of the time from which we count the events + */ + private async getWorkspaceEventsCount( + workspace: WorkspaceWithTariffPlan, + projects: ProjectDBScheme[], + since: number + ): Promise { + if (!this.shouldUseDailyEventsCounter(workspace._id.toString())) { + return this.dbHelper.getEventsCountByProjects(projects, since); + } + + const oldAlgoStartedAt = Date.now(); + const oldAlgoCount = await this.dbHelper.getEventsCountByProjects(projects, since); + const oldAlgoTook = (Date.now() - oldAlgoStartedAt) / MS_IN_SEC; + + try { + const newAlgoStartedAt = Date.now(); + const newAlgoCount = await this.dbHelper.getEventsCountByProjectsUsingDailyEvents(projects, since); + const newAlgoTook = (Date.now() - newAlgoStartedAt) / MS_IN_SEC; + + telegram.sendMessage( + `Workspace ${workspace.name} event count:\n` + + `Old algo: ${oldAlgoCount}, took ${oldAlgoTook}sec\n` + + `New algo: ${newAlgoCount}, took ${newAlgoTook}sec`, + telegram.TelegramBotURLs.Limiter + ); + + return newAlgoCount; + } catch (error) { + HawkCatcher.send(error, { + workspaceId: workspace._id.toString(), + }); + + return oldAlgoCount; + } + } + + /** + * Checks whether dailyEvents-based quota counting is enabled for the workspace + * via LIMITER_DAILY_EVENTS_COUNTER_WORKSPACE_IDS environment variable — + * comma-separated workspace ids or `*` to enable it for every workspace. + * + * @param workspaceId - workspace id + */ + private shouldUseDailyEventsCounter(workspaceId: string): boolean { + const enabledWorkspaceIds = (process.env.LIMITER_DAILY_EVENTS_COUNTER_WORKSPACE_IDS || '') + .split(',') + .map(id => id.trim()) + .filter(Boolean); + + return enabledWorkspaceIds.includes('*') || enabledWorkspaceIds.includes(workspaceId); + } + /** * Method that formats project list to html used in report messages * diff --git a/workers/limiter/tests/dbHelper.test.ts b/workers/limiter/tests/dbHelper.test.ts index f57be08d..dd26baa7 100644 --- a/workers/limiter/tests/dbHelper.test.ts +++ b/workers/limiter/tests/dbHelper.test.ts @@ -9,9 +9,20 @@ import HawkCatcher from '@hawk.so/nodejs'; /** * Constant of last charge date in all workspaces for tests + * 2020-04-01T12:00:00Z — intentionally not midnight-aligned */ const LAST_CHARGE_DATE = new Date(1585742400 * 1000); +/** + * Timestamp inside the boundary day of LAST_CHARGE_DATE (2020-04-01T16:00:00Z) + */ +const BOUNDARY_DAY_TIMESTAMP = 1585756800; + +/** + * UTC midnight right after LAST_CHARGE_DATE (2020-04-02T00:00:00Z) + */ +const NEXT_MIDNIGHT_AFTER_LAST_CHARGE = 1585785600; + describe('DbHelper', () => { let connection: MongoClient; let db: Db; @@ -66,8 +77,10 @@ describe('DbHelper', () => { /** * Returns mocked event for tests + * + * @param timestamp - event timestamp, defaults to the boundary day of LAST_CHARGE_DATE */ - const createEventMock = (): GroupedEventDBScheme => { + const createEventMock = (timestamp: number = BOUNDARY_DAY_TIMESTAMP): GroupedEventDBScheme => { return { catcherType: '', totalCount: 0, @@ -77,7 +90,7 @@ describe('DbHelper', () => { payload: { title: 'Mocked event', }, - timestamp: 1586892935, + timestamp, }; }; @@ -89,11 +102,13 @@ describe('DbHelper', () => { const fillDatabaseWithMockedData = async (parameters: { workspace?: WorkspaceDBScheme, project: ProjectDBScheme, - eventsToMock: number + eventsToMock: number, repetitionsToMock?: number, + dailyEventsToMock?: Array<{ groupingTimestamp: number; count: number }>, }): Promise => { const eventsCollection = db.collection(`events:${parameters.project._id.toString()}`); const repetitionsCollection = db.collection(`repetitions:${parameters.project._id.toString()}`); + const dailyEventsCollection = db.collection(`dailyEvents:${parameters.project._id.toString()}`); if (parameters.workspace) { await workspaceCollection.insertOne(parameters.workspace); @@ -104,7 +119,9 @@ describe('DbHelper', () => { for (let i = 0; i < parameters.eventsToMock; i++) { mockedEvents.push(createEventMock()); } - await eventsCollection.insertMany(mockedEvents); + if (mockedEvents.length > 0) { + await eventsCollection.insertMany(mockedEvents); + } mockedEvents.length = 0; @@ -114,6 +131,14 @@ describe('DbHelper', () => { } await repetitionsCollection.insertMany(mockedEvents); } + + if (parameters.dailyEventsToMock?.length > 0) { + await dailyEventsCollection.insertMany(parameters.dailyEventsToMock.map(bucket => ({ + groupHash: 'ade987831d0d0d167aeea685b49db164eb4e113fd027858eef7f69d049357f62', + groupingTimestamp: bucket.groupingTimestamp, + count: bucket.count, + }))); + } }; beforeAll(async () => { @@ -554,7 +579,7 @@ describe('DbHelper', () => { }); describe('getEventsCountByProject', () => { - test('Should count events and repetitions for a project', async () => { + test('Should count boundary-day events and repetitions for a project', async () => { /** * Arrange */ @@ -584,10 +609,180 @@ describe('DbHelper', () => { */ expect(count).toBe(10); // 5 events + 5 repetitions }); + + test('Should keep raw counting as default and include raw docs after the boundary day', async () => { + /** + * Arrange + */ + const workspace = createWorkspaceMock({ + plan: mockedPlans.eventsLimit10, + billingPeriodEventsCount: 0, + lastChargeDate: new Date(), + }); + const project = createProjectMock({ workspaceId: workspace._id }); + const since = Math.floor(LAST_CHARGE_DATE.getTime() / MS_IN_SEC); + + await fillDatabaseWithMockedData({ + workspace, + project, + eventsToMock: 1, + dailyEventsToMock: [ + { + groupingTimestamp: NEXT_MIDNIGHT_AFTER_LAST_CHARGE, + count: 100, + }, + ], + }); + + await db.collection(`events:${project._id.toString()}`).insertOne( + createEventMock(NEXT_MIDNIGHT_AFTER_LAST_CHARGE + 100) + ); + + /** + * Act + */ + const count = await dbHelper.getEventsCountByProject(project, since); + + /** + * Assert + */ + expect(count).toBe(2); + }); + }); + + describe('getEventsCountByProjectUsingDailyEvents', () => { + test('Should add per-day counters from dailyEvents for days after the boundary day', async () => { + /** + * Arrange + */ + const workspace = createWorkspaceMock({ + plan: mockedPlans.eventsLimit10, + billingPeriodEventsCount: 0, + lastChargeDate: new Date(), + }); + const project = createProjectMock({ workspaceId: workspace._id }); + + await fillDatabaseWithMockedData({ + workspace, + project, + eventsToMock: 2, + repetitionsToMock: 3, + dailyEventsToMock: [ + { + groupingTimestamp: NEXT_MIDNIGHT_AFTER_LAST_CHARGE, + count: 7, + }, + { + groupingTimestamp: NEXT_MIDNIGHT_AFTER_LAST_CHARGE + 86400, + count: 3, + }, + ], + }); + + const since = Math.floor(LAST_CHARGE_DATE.getTime() / MS_IN_SEC); + + /** + * Act + */ + const count = await dbHelper.getEventsCountByProjectUsingDailyEvents(project, since); + + /** + * Assert + */ + expect(count).toBe(15); // 2 events + 3 repetitions on the boundary day + 7 + 3 from dailyEvents + }); + + test('Should ignore raw docs outside the boundary day and dailyEvents buckets before it', async () => { + /** + * Arrange + */ + const workspace = createWorkspaceMock({ + plan: mockedPlans.eventsLimit10, + billingPeriodEventsCount: 0, + lastChargeDate: new Date(), + }); + const project = createProjectMock({ workspaceId: workspace._id }); + const since = Math.floor(LAST_CHARGE_DATE.getTime() / MS_IN_SEC); + + await fillDatabaseWithMockedData({ + workspace, + project, + eventsToMock: 1, + dailyEventsToMock: [ + /** bucket of the boundary day itself must not be counted */ + { + groupingTimestamp: NEXT_MIDNIGHT_AFTER_LAST_CHARGE - 86400, + count: 100, + }, + ], + }); + + const eventsCollection = db.collection(`events:${project._id.toString()}`); + + await eventsCollection.insertMany([ + /** before lastChargeDate */ + createEventMock(since - 100), + /** after the boundary day — counted via dailyEvents, not the raw scan */ + createEventMock(NEXT_MIDNIGHT_AFTER_LAST_CHARGE + 100), + ]); + + /** + * Act + */ + const count = await dbHelper.getEventsCountByProjectUsingDailyEvents(project, since); + + /** + * Assert + */ + expect(count).toBe(1); // only the single boundary-day event + }); + + test('Should count dailyEvents bucket at since when since is already UTC midnight', async () => { + /** + * Arrange + */ + const workspace = createWorkspaceMock({ + plan: mockedPlans.eventsLimit10, + billingPeriodEventsCount: 0, + lastChargeDate: new Date(), + }); + const project = createProjectMock({ workspaceId: workspace._id }); + const since = NEXT_MIDNIGHT_AFTER_LAST_CHARGE; + + await fillDatabaseWithMockedData({ + workspace, + project, + eventsToMock: 0, + dailyEventsToMock: [ + { + groupingTimestamp: since, + count: 4, + }, + { + groupingTimestamp: since + 86400, + count: 3, + }, + ], + }); + + await db.collection(`events:${project._id.toString()}`).insertOne( + createEventMock(since + 100) + ); + + /** + * Act + */ + const count = await dbHelper.getEventsCountByProjectUsingDailyEvents(project, since); + + /** + * Assert + */ + expect(count).toBe(7); + }); }); - describe('getEventsCountByProjects', () => { - test('Should count events and repetitions for multiple projects', async () => { + describe('getEventsCountByProjectsUsingDailyEvents', () => { + test('Should count events, repetitions and dailyEvents for multiple projects', async () => { /** * Arrange */ @@ -604,6 +799,12 @@ describe('DbHelper', () => { project: project1, eventsToMock: 5, repetitionsToMock: 5, + dailyEventsToMock: [ + { + groupingTimestamp: NEXT_MIDNIGHT_AFTER_LAST_CHARGE, + count: 6, + }, + ], }); await fillDatabaseWithMockedData({ project: project2, @@ -616,12 +817,12 @@ describe('DbHelper', () => { /** * Act */ - const count = await dbHelper.getEventsCountByProjects([project1, project2], since); + const count = await dbHelper.getEventsCountByProjectsUsingDailyEvents([project1, project2], since); /** * Assert */ - expect(count).toBe(16); // (5 + 5) + (3 + 3) events and repetitions + expect(count).toBe(22); // (5 + 5 + 6) + (3 + 3) }); }); diff --git a/workers/limiter/tests/index.test.ts b/workers/limiter/tests/index.test.ts index 8a7ebf7b..1c6c7648 100644 --- a/workers/limiter/tests/index.test.ts +++ b/workers/limiter/tests/index.test.ts @@ -27,6 +27,17 @@ const REGULAR_WORKSPACES_CHECK_EVENT: RegularWorkspacesCheckEvent = { */ const LAST_CHARGE_DATE = new Date(1585742400 * 1000); +/** + * Timestamp inside the boundary day of LAST_CHARGE_DATE (2020-04-01T16:00:00Z): + * such events are counted from the raw collections, later ones — via dailyEvents + */ +const BOUNDARY_DAY_TIMESTAMP = 1585756800; + +/** + * UTC midnight right after LAST_CHARGE_DATE (2020-04-02T00:00:00Z) + */ +const NEXT_MIDNIGHT_AFTER_LAST_CHARGE = 1585785600; + describe('Limiter worker', () => { let connection: MongoClient; let db: Db; @@ -89,7 +100,7 @@ describe('Limiter worker', () => { usersAffected: 0, visitedBy: [], groupHash: 'ade987831d0d0d167aeea685b49db164eb4e113fd027858eef7f69d049357f62', - timestamp: 1586892935, + timestamp: BOUNDARY_DAY_TIMESTAMP, payload: { title: 'Mocked event', }, @@ -104,7 +115,7 @@ describe('Limiter worker', () => { const fillDatabaseWithMockedData = async (parameters: { workspace: WorkspaceDBScheme, project: ProjectDBScheme, - eventsToMock: number + eventsToMock: number, repetitionsToMock?: number, }): Promise => { const eventsCollection = db.collection(`events:${parameters.project._id.toString()}`); @@ -320,6 +331,65 @@ describe('Limiter worker', () => { expect(reportMessage).toContain(`${project1.name} (id: ${project1._id})`); }); + test('Should compute both counters and report the comparison to Telegram when dailyEvents counter is enabled', async () => { + /** + * Arrange + */ + const workspace = createWorkspaceMock({ + plan: mockedPlans.eventsLimit10000, + billingPeriodEventsCount: 0, + lastChargeDate: LAST_CHARGE_DATE, + }); + const project = createProjectMock({ workspaceId: workspace._id }); + + await fillDatabaseWithMockedData({ + workspace, + project, + eventsToMock: 5, + }); + + /** + * Bucket for the day after the boundary day — counted only by the new algorithm + */ + await db.collection(`dailyEvents:${project._id.toString()}`).insertOne({ + groupHash: 'ade987831d0d0d167aeea685b49db164eb4e113fd027858eef7f69d049357f62', + groupingTimestamp: NEXT_MIDNIGHT_AFTER_LAST_CHARGE, + count: 7, + }); + + process.env.LIMITER_DAILY_EVENTS_COUNTER_WORKSPACE_IDS = workspace._id.toString(); + + /** + * Act + */ + try { + const worker = new LimiterWorker(); + + await worker.start(); + await worker.handle(REGULAR_WORKSPACES_CHECK_EVENT); + await worker.finish(); + } finally { + delete process.env.LIMITER_DAILY_EVENTS_COUNTER_WORKSPACE_IDS; + } + + /** + * Assert — the new counter result is saved, both results are reported with timings + */ + const workspaceInDatabase = await workspaceCollection.findOne({ + _id: workspace._id, + }); + + expect(workspaceInDatabase.billingPeriodEventsCount).toBe(12); // 5 boundary-day events + 7 from dailyEvents + + const comparisonMessage = (telegram.sendMessage as jest.Mock).mock.calls + .map(call => call[0]) + .find(message => message.includes('Old algo')); + + expect(comparisonMessage).toContain(`Workspace ${workspace.name} event count:`); + expect(comparisonMessage).toMatch(/Old algo: 5, took [\d.]+sec/); + expect(comparisonMessage).toMatch(/New algo: 12, took [\d.]+sec/); + }); + test('Should not send a report when no projects are blocked or unblocked', async () => { /** * Arrange diff --git a/workers/sender/src/index.ts b/workers/sender/src/index.ts index 242d2da8..a52b05c5 100644 --- a/workers/sender/src/index.ts +++ b/workers/sender/src/index.ts @@ -192,7 +192,7 @@ export default abstract class SenderWorker extends Worker { this.logger.info(`Sending ${notificationType} notification to ${channel.endpoint}`); - this.provider.send(channel.endpoint, { + await this.provider.send(channel.endpoint, { type: notificationType, payload: { host: process.env.GARAGE_URL, @@ -245,7 +245,7 @@ export default abstract class SenderWorker extends Worker { return; } - this.provider.send(endpoint, { + await this.provider.send(endpoint, { type: 'assignee', payload: { host: process.env.GARAGE_URL, @@ -506,7 +506,7 @@ export default abstract class SenderWorker extends Worker { return; } - this.provider.send(endpoint, { + await this.provider.send(endpoint, { type: 'payment-failed', payload: { host: process.env.GARAGE_URL, @@ -541,7 +541,7 @@ export default abstract class SenderWorker extends Worker { return; } - this.provider.send(endpoint, { + await this.provider.send(endpoint, { type: 'payment-success', payload: { host: process.env.GARAGE_URL, @@ -560,7 +560,7 @@ export default abstract class SenderWorker extends Worker { private async handlePasswordResetTask(task: SenderWorkerPasswordResetTask): Promise { const { newPassword, endpoint } = task.payload; - this.provider.send(endpoint, { + await this.provider.send(endpoint, { type: 'password-reset', payload: { host: process.env.GARAGE_URL, @@ -578,7 +578,7 @@ export default abstract class SenderWorker extends Worker { private async handleWorkspaceInviteTask(task: SenderWorkerWorkspaceInviteTask): Promise { const { workspaceName, inviteLink, endpoint } = task.payload; - this.provider.send(endpoint, { + await this.provider.send(endpoint, { type: 'workspace-invite', payload: { host: process.env.GARAGE_URL, @@ -597,7 +597,7 @@ export default abstract class SenderWorker extends Worker { private async handleSignUpTask(task: SenderWorkerSignUpTask): Promise { const { password, endpoint } = task.payload; - this.provider.send(endpoint, { + await this.provider.send(endpoint, { type: 'sign-up', payload: { host: process.env.GARAGE_URL, diff --git a/workers/sender/tests/worker.test.ts b/workers/sender/tests/worker.test.ts index 73a9b5ae..ef5b0512 100644 --- a/workers/sender/tests/worker.test.ts +++ b/workers/sender/tests/worker.test.ts @@ -249,4 +249,33 @@ describe('Sender Worker', () => { expect(dailyEventsQueryMock).toBeCalledWith({ groupHash: 'groupHash' }); }); }); + + describe('provider.send awaiting', () => { + /** + * Without await, handle() resolves even if send() rejects — + * message would be acked and the error lost. + * The .catch on the rejected promise only prevents an unhandledRejection + * crash in the fire-and-forget case; await still observes the rejection. + */ + it('should reject handle when provider.send fails', async () => { + const worker = new ExampleSenderWorker(); + const sendError = new Error('provider send failed'); + + (worker as any).provider.send = jest.fn(() => { + const sendPromise = Promise.reject(sendError); + + sendPromise.catch(() => undefined); + + return sendPromise; + }); + + await expect(worker.handle({ + type: 'sign-up', + payload: { + password: 'secret', + endpoint: 'user@example.com', + }, + })).rejects.toThrow('provider send failed'); + }); + }); });