Update prod

.env.sample

@@ -55,5 +55,8 @@ HAWK_CATCHER_TOKEN=
 ## If true, Grouper worker will send messages about new events to Notifier worker
 IS_NOTIFIER_WORKER_ENABLED=false
 
+## Comma-separated workspace ids that should use dailyEvents counters in Limiter quota checks. Use * for all workspaces.
+LIMITER_DAILY_EVENTS_COUNTER_WORKSPACE_IDS=
+
 ## Url for telegram notifications about workspace blocks and unblocks 
 TELEGRAM_LIMITER_CHAT_URL=

lib/db/controller.test.ts

@@ -1,5 +1,6 @@
 import * as mongodb from 'mongodb';
 import { DatabaseController } from './controller';
+import { DatabaseConnectionError } from '../workerErrors';
 import '../../env-test';
 
 /**
@@ -24,4 +25,47 @@ describe('Database Controller Test', () => {
       expect(result).toBe(true);
     });
   });
+
+  describe('initial handshake retry', () => {
+    const mongoModule = jest.requireActual('mongodb');
+    let connectSpy: jest.SpyInstance;
+
+    beforeEach(() => {
+      process.env.MONGO_RECONNECT_TRIES = '5';
+      process.env.MONGO_RECONNECT_INTERVAL = '1';
+      jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+      connectSpy = jest.spyOn(mongoModule, 'connect');
+    });
+
+    afterEach(() => {
+      delete process.env.MONGO_RECONNECT_TRIES;
+      delete process.env.MONGO_RECONNECT_INTERVAL;
+      jest.restoreAllMocks();
+    });
+
+    it('retries the initial connection until it succeeds', async () => {
+      const fakeDb = {} as mongodb.Db;
+      const fakeClient = { db: jest.fn().mockReturnValue(fakeDb) } as unknown as mongodb.MongoClient;
+
+      connectSpy
+        .mockRejectedValueOnce(new Error('unreachable'))
+        .mockRejectedValueOnce(new Error('unreachable'))
+        .mockResolvedValueOnce(fakeClient);
+
+      const controller = new DatabaseController('mongodb://localhost:27017/test');
+      const result = await controller.connect();
+
+      expect(connectSpy).toHaveBeenCalledTimes(3);
+      expect(result).toBe(fakeDb);
+    });
+
+    it('throws DatabaseConnectionError after exhausting all retries', async () => {
+      connectSpy.mockRejectedValue(new Error('unreachable'));
+
+      const controller = new DatabaseController('mongodb://localhost:27017/test');
+
+      await expect(controller.connect()).rejects.toBeInstanceOf(DatabaseConnectionError);
+      expect(connectSpy).toHaveBeenCalledTimes(5);
+    });
+  });
 });

lib/db/controller.ts

@@ -1,5 +1,22 @@
 import { GridFSBucket, MongoClient, Db, connect } from 'mongodb';
 import { DatabaseConnectionError } from '../workerErrors';
+import { positiveIntEnv } from '../utils/positiveIntEnv';
+
+/**
+ * How many times to retry the initial Mongo handshake before giving up
+ */
+const DEFAULT_RECONNECT_TRIES = 60;
+
+/**
+ * Delay between initial-handshake retries, in ms
+ */
+const DEFAULT_RECONNECT_INTERVAL_MS = 3000;
+
+/**
+ * Bounds how long a single attempt waits for an available server, so a retry
+ * fails fast during an outage instead of hanging on the 30s driver default
+ */
+const SERVER_SELECTION_TIMEOUT_MS = 10000;
 
 /**
  * Database connection singleton
@@ -46,28 +63,50 @@ export class DatabaseController {
   }
 
   /**
-   * Connect to database
-   * Requires `MONGO_DSN` environment variable to be set
+   * Connect to the database, retrying with a fixed backoff while the server is
+   * unreachable so a worker booting during a Mongo outage waits instead of
+   * crash-looping. The driver auto-recovers already-open connections on its
+   * own, so this retry covers the initial handshake only.
    *
-   * @throws {Error} if `MONGO_DSN` is not set
+   * Tunable via MONGO_RECONNECT_TRIES (default 60) and
+   * MONGO_RECONNECT_INTERVAL in ms (default 3000).
+   *
+   * @throws {DatabaseConnectionError} if every attempt fails
    */
   public async connect(): Promise<Db> {
     if (this.db) {
-      return;
+      return this.db;
     }
 
-    try {
-      this.connection = await connect(this.connectionUri, {
-        useNewUrlParser: true,
-        useUnifiedTopology: true,
-        ...(this.appName ? { appName: this.appName } : {}),
-      });
-      this.db = await this.connection.db();
+    const tries = positiveIntEnv(process.env.MONGO_RECONNECT_TRIES, DEFAULT_RECONNECT_TRIES);
+    const intervalMs = positiveIntEnv(process.env.MONGO_RECONNECT_INTERVAL, DEFAULT_RECONNECT_INTERVAL_MS);
 
-      return this.db;
-    } catch (err) {
-      throw new DatabaseConnectionError(err);
+    for (let attempt = 1; attempt <= tries; attempt++) {
+      try {
+        this.connection = await connect(this.connectionUri, {
+          useNewUrlParser: true,
+          useUnifiedTopology: true,
+          serverSelectionTimeoutMS: SERVER_SELECTION_TIMEOUT_MS,
+          /** driver 3.x only recognizes the lowercase `appname` option key */
+          ...(this.appName ? { appname: this.appName } : {}),
+        });
+        this.db = this.connection.db();
+
+        return this.db;
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+
+        console.warn(`[Mongo] connect attempt ${attempt}/${tries} failed: ${message}`);
+
+        if (attempt >= tries) {
+          throw new DatabaseConnectionError(err);
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, intervalMs));
+      }
     }
+
+    throw new DatabaseConnectionError('Failed to connect to MongoDB');
   }
 
   /**

lib/utils/positiveIntEnv.ts

@@ -0,0 +1,16 @@
+/**
+ * Parses a positive-integer env var, using `fallback` for missing, non-numeric,
+ * zero or negative values
+ *
+ * @param value - raw env var value
+ * @param fallback - default for an invalid value
+ */
+export function positiveIntEnv(value: string | undefined, fallback: number): number {
+  const parsed = Number(value);
+
+  if (!Number.isFinite(parsed) || parsed < 1) {
+    return fallback;
+  }
+
+  return Math.floor(parsed);
+}

lib/utils/sanitizer.ts

@@ -0,0 +1,129 @@
+import { rightTrim } from './string';
+
+/**
+ * Maximum string length before appending ellipsis
+ */
+const MAX_STRING_LENGTH = 200;
+
+/**
+ * Maximum number of object keys to keep, the rest are reported via META_FIELD
+ */
+const MAX_OBJECT_KEYS_COUNT = 20;
+
+/**
+ * Key added to an object to report how many of its keys were skipped
+ */
+const META_FIELD = '__meta';
+
+/**
+ * Maximum depth of sanitized objects
+ */
+const MAX_DEPTH = 5;
+
+/**
+ * Maximum length of sanitized arrays
+ */
+const MAX_ARRAY_LENGTH = 10;
+
+/**
+ * Checks that the value is a plain object
+ *
+ * @param target - value to check
+ */
+function isPlainObject(target: unknown): target is Record<string, unknown> {
+  return Object.prototype.toString.call(target) === '[object Object]';
+}
+
+/**
+ * Values that can be tracked by WeakSet to detect circular references
+ */
+type ObjectLike = Record<string, unknown> | unknown[];
+
+/**
+ * Prepares event data for storing: trims long strings, slices long arrays,
+ * cuts off extra object keys and replaces too deep objects and circular
+ * references with placeholders.
+ */
+export class Sanitizer {
+  /**
+   * Apply sanitizing for array/object/primitives
+   *
+   * @param data - any value to sanitize
+   * @param depth - current depth of recursion
+   * @param seen - already visited objects
+   */
+  public static sanitize(data: unknown, depth = 0, seen = new WeakSet<ObjectLike>()): unknown {
+    if (data !== null && typeof data === 'object') {
+      if (seen.has(data as ObjectLike)) {
+        return '<circular>';
+      }
+      seen.add(data as ObjectLike);
+    }
+
+    if (Array.isArray(data)) {
+      return Sanitizer.sanitizeArray(data, depth + 1, seen);
+    }
+
+    if (isPlainObject(data)) {
+      return Sanitizer.sanitizeObject(data, depth + 1, seen);
+    }
+
+    if (typeof data === 'string') {
+      return rightTrim(data, MAX_STRING_LENGTH);
+    }
+
+    return data;
+  }
+
+  /**
+   * Slices array to the maximum length and sanitizes each element
+   *
+   * @param arr - array to sanitize
+   * @param depth - current depth of recursion
+   * @param seen - already visited objects
+   */
+  private static sanitizeArray(arr: unknown[], depth: number, seen: WeakSet<ObjectLike>): unknown[] {
+    const length = arr.length;
+
+    if (length > MAX_ARRAY_LENGTH) {
+      arr = arr.slice(0, MAX_ARRAY_LENGTH);
+      arr.push(`<${length - MAX_ARRAY_LENGTH} more items...>`);
+    }
+
+    return arr.map((item) => {
+      return Sanitizer.sanitize(item, depth, seen);
+    });
+  }
+
+  /**
+   * Sanitizes object values recursively
+   *
+   * @param data - object to sanitize
+   * @param depth - current depth of recursion
+   * @param seen - already visited objects
+   */
+  private static sanitizeObject(
+    data: Record<string, unknown>,
+    depth: number,
+    seen: WeakSet<ObjectLike>
+  ): Record<string, unknown> | '<deep object>' {
+    if (depth > MAX_DEPTH) {
+      return '<deep object>';
+    }
+
+    const keys = Object.keys(data);
+    const result: Record<string, unknown> = {};
+
+    for (const key of keys.slice(0, MAX_OBJECT_KEYS_COUNT)) {
+      result[key] = Sanitizer.sanitize(data[key], depth, seen);
+    }
+
+    const skippedKeysCount = keys.length - MAX_OBJECT_KEYS_COUNT;
+
+    if (skippedKeysCount > 0) {
+      result[META_FIELD] = `${skippedKeysCount} more key(s) skipped`;
+    }
+
+    return result;
+  }
+}