Merge master into prod

Author: Kuchizu

.env.sample

@@ -55,5 +55,8 @@ HAWK_CATCHER_TOKEN=
 ## If true, Grouper worker will send messages about new events to Notifier worker
 IS_NOTIFIER_WORKER_ENABLED=false
 
+## Comma-separated workspace ids that should use dailyEvents counters in Limiter quota checks. Use * for all workspaces.
+LIMITER_DAILY_EVENTS_COUNTER_WORKSPACE_IDS=
+
 ## Url for telegram notifications about workspace blocks and unblocks 
 TELEGRAM_LIMITER_CHAT_URL=

lib/db/controller.ts

@@ -87,7 +87,8 @@ export class DatabaseController {
           useNewUrlParser: true,
           useUnifiedTopology: true,
           serverSelectionTimeoutMS: SERVER_SELECTION_TIMEOUT_MS,
-          ...(this.appName ? { appName: this.appName } : {}),
+          /** driver 3.x only recognizes the lowercase `appname` option key */
+          ...(this.appName ? { appname: this.appName } : {}),
         });
         this.db = this.connection.db();
 

lib/utils/sanitizer.ts

@@ -0,0 +1,129 @@
+import { rightTrim } from './string';
+
+/**
+ * Maximum string length before appending ellipsis
+ */
+const MAX_STRING_LENGTH = 200;
+
+/**
+ * Maximum number of object keys to keep, the rest are reported via META_FIELD
+ */
+const MAX_OBJECT_KEYS_COUNT = 20;
+
+/**
+ * Key added to an object to report how many of its keys were skipped
+ */
+const META_FIELD = '__meta';
+
+/**
+ * Maximum depth of sanitized objects
+ */
+const MAX_DEPTH = 5;
+
+/**
+ * Maximum length of sanitized arrays
+ */
+const MAX_ARRAY_LENGTH = 10;
+
+/**
+ * Checks that the value is a plain object
+ *
+ * @param target - value to check
+ */
+function isPlainObject(target: unknown): target is Record<string, unknown> {
+  return Object.prototype.toString.call(target) === '[object Object]';
+}
+
+/**
+ * Values that can be tracked by WeakSet to detect circular references
+ */
+type ObjectLike = Record<string, unknown> | unknown[];
+
+/**
+ * Prepares event data for storing: trims long strings, slices long arrays,
+ * cuts off extra object keys and replaces too deep objects and circular
+ * references with placeholders.
+ */
+export class Sanitizer {
+  /**
+   * Apply sanitizing for array/object/primitives
+   *
+   * @param data - any value to sanitize
+   * @param depth - current depth of recursion
+   * @param seen - already visited objects
+   */
+  public static sanitize(data: unknown, depth = 0, seen = new WeakSet<ObjectLike>()): unknown {
+    if (data !== null && typeof data === 'object') {
+      if (seen.has(data as ObjectLike)) {
+        return '<circular>';
+      }
+      seen.add(data as ObjectLike);
+    }
+
+    if (Array.isArray(data)) {
+      return Sanitizer.sanitizeArray(data, depth + 1, seen);
+    }
+
+    if (isPlainObject(data)) {
+      return Sanitizer.sanitizeObject(data, depth + 1, seen);
+    }
+
+    if (typeof data === 'string') {
+      return rightTrim(data, MAX_STRING_LENGTH);
+    }
+
+    return data;
+  }
+
+  /**
+   * Slices array to the maximum length and sanitizes each element
+   *
+   * @param arr - array to sanitize
+   * @param depth - current depth of recursion
+   * @param seen - already visited objects
+   */
+  private static sanitizeArray(arr: unknown[], depth: number, seen: WeakSet<ObjectLike>): unknown[] {
+    const length = arr.length;
+
+    if (length > MAX_ARRAY_LENGTH) {
+      arr = arr.slice(0, MAX_ARRAY_LENGTH);
+      arr.push(`<${length - MAX_ARRAY_LENGTH} more items...>`);
+    }
+
+    return arr.map((item) => {
+      return Sanitizer.sanitize(item, depth, seen);
+    });
+  }
+
+  /**
+   * Sanitizes object values recursively
+   *
+   * @param data - object to sanitize
+   * @param depth - current depth of recursion
+   * @param seen - already visited objects
+   */
+  private static sanitizeObject(
+    data: Record<string, unknown>,
+    depth: number,
+    seen: WeakSet<ObjectLike>
+  ): Record<string, unknown> | '<deep object>' {
+    if (depth > MAX_DEPTH) {
+      return '<deep object>';
+    }
+
+    const keys = Object.keys(data);
+    const result: Record<string, unknown> = {};
+
+    for (const key of keys.slice(0, MAX_OBJECT_KEYS_COUNT)) {
+      result[key] = Sanitizer.sanitize(data[key], depth, seen);
+    }
+
+    const skippedKeysCount = keys.length - MAX_OBJECT_KEYS_COUNT;
+
+    if (skippedKeysCount > 0) {
+      result[META_FIELD] = `${skippedKeysCount} more key(s) skipped`;
+    }
+
+    return result;
+  }
+}

workers/grouper/src/data-filter.ts

@@ -1,11 +1,18 @@
 import type { EventAddons, EventData } from '@hawk.so/types';
 import { unsafeFields } from '../../../lib/utils/unsafeFields';
+import { rightTrim } from '../../../lib/utils/string';
+import { Sanitizer } from '../../../lib/utils/sanitizer';
 
 /**
  * Maximum depth for object traversal to prevent excessive memory allocations
  */
 const MAX_TRAVERSAL_DEPTH = 20;
 
+/**
+ * Maximum length for event title before appending ellipsis
+ */
+const MAX_TITLE_LENGTH = 400;
+
 /**
  * Recursively iterate through object and call function on each key
  *
@@ -135,13 +142,58 @@ export default class DataFilter {
    * @param event - event to process
    */
   public processEvent(event: EventData<EventAddons>): void {
+    this.trimEventTitle(event);
+    this.sanitizeEvent(event);
+
     unsafeFields.forEach(field => {
       if (event[field]) {
         this.processField(event[field]);
       }
     });
   }
 
+  /**
+   * Trim event title to the maximum allowed length.
+   * It mutates the original object.
+   *
+   * @param event - event to process
+   */
+  public trimEventTitle(event: EventData<EventAddons>): void {
+    if (typeof event.title === 'string') {
+      event.title = rightTrim(event.title, MAX_TITLE_LENGTH);
+    }
+  }
+
+  /**
+   * Sanitize event fields that can contain long strings, deep objects or long arrays.
+   * It mutates the original object.
+   *
+   * @param event - event to process
+   */
+  public sanitizeEvent(event: EventData<EventAddons>): void {
+    unsafeFields.forEach(field => {
+      if (event[field] !== undefined) {
+        (event as unknown as Record<string, unknown>)[field] = Sanitizer.sanitize(event[field]);
+      }
+    });
+
+    event.backtrace?.forEach(frame => {
+      if (frame.arguments !== undefined) {
+        frame.arguments = Sanitizer.sanitize(frame.arguments) as string[];
+      }
+    });
+
+    event.breadcrumbs?.forEach(breadcrumb => {
+      if (typeof breadcrumb.message === 'string') {
+        breadcrumb.message = Sanitizer.sanitize(breadcrumb.message) as string;
+      }
+
+      if (breadcrumb.data !== undefined) {
+        breadcrumb.data = Sanitizer.sanitize(breadcrumb.data) as typeof breadcrumb.data;
+      }
+    });
+  }
+
   /**
    * Recursively iterates object and applies filtering to its entries
    *

workers/grouper/src/index.ts

@@ -59,7 +59,7 @@ const DB_DUPLICATE_KEY_ERROR = '11000';
 const DAILY_METRICS_RETENTION_DAYS = 90;
 
 /**
- * Maximum length for backtrace code line or title
+ * Maximum length for backtrace code line
  */
 const MAX_CODE_LINE_LENGTH = 140;
 
@@ -198,8 +198,6 @@ export default class GrouperWorker extends Worker {
     this.grouperMetrics.observePayloadSize(taskPayloadSize);
     this.memoryMonitor.logBeforeHandle(memoryBeforeHandle, handledTasksCount, taskPayloadSize, task.projectId);
 
-    this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`);
-
     // FIX RELEASE TYPE
     // TODO: REMOVE AFTER 01.01.2026, after the most of the users update to new js catcher
     if (task.payload && task.payload.release !== undefined) {
@@ -209,6 +207,11 @@ export default class GrouperWorker extends Worker {
       };
     }
 
+    /**
+     * Filter event data before hashing so hash and stored event stay consistent.
+     */
+    this.dataFilter.processEvent(task.payload);
+
     let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task));
     let existedEvent: GroupedEventDBScheme;
     let repetitionId = null;
@@ -219,11 +222,6 @@ export default class GrouperWorker extends Worker {
        * Trim source code lines to prevent memory leaks
        */
       this.trimSourceCodeLines(task.payload);
-
-      /**
-       * Filter sensitive information
-       */
-      this.dataFilter.processEvent(task.payload);
     });
 
     /**