feat(replay): Throttle breadcrumbs to max 300/5s (#8086)

This updates custom breadcrumb handling to be throttled to max. 300 breadcrumbs/5s. If we exceed this amount of breadcrumbs, we drop any further breadcrumbs and instead add a single breadcrumb with `category: 'replay.throttled'`, which the UI can use to indicate that _something_ was dropped. Closes #8072 --------- Co-authored-by: Billy Vong <billyvg@users.noreply.github.com>
getsentry · May 26, 2023 · f359ef3 · f359ef3
1 parent 41fef4b
commit f359ef3
Show file tree

Hide file tree

Showing 12 changed files with 344 additions and 18 deletions.
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/init.js b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/init.js
@@ -0,0 +1,17 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+window.Replay = new Sentry.Replay({
+  flushMinDelay: 5000,
+  flushMaxDelay: 5000,
+  useCompression: false,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 0,
+  replaysSessionSampleRate: 1.0,
+  replaysOnErrorSampleRate: 0.0,
+
+  integrations: [window.Replay],
+});
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/subject.js b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/subject.js
@@ -0,0 +1,8 @@
+const COUNT = 400;
+
+document.querySelector('[data-console]').addEventListener('click', () => {
+  // Call console.log() many times
+  for (let i = 0; i < COUNT; i++) {
+    console.log(`testing ${i}`);
+  }
+});
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/template.html b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/template.html
@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+  </head>
+  <body>
+    <button data-console>Trigger console.log</button>
+  </body>
+</html>
diff --git a/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/test.ts b/packages/browser-integration-tests/suites/replay/throttleBreadcrumbs/test.ts
@@ -0,0 +1,41 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../utils/fixtures';
+import { getCustomRecordingEvents, shouldSkipReplayTest, waitForReplayRequest } from '../../../utils/replayHelpers';
+
+const THROTTLE_LIMIT = 300;
+
+sentryTest(
+  'throttles breadcrumbs when many `console.log` are made at the same time',
+  async ({ getLocalTestUrl, page, forceFlushReplay, browserName }) => {
+    if (shouldSkipReplayTest() || browserName !== 'chromium') {
+      sentryTest.skip();
+    }
+
+    const reqPromise0 = waitForReplayRequest(page, 0);
+    const reqPromise1 = waitForReplayRequest(page, 1);
+
+    await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+      return route.fulfill({
+        status: 200,
+        contentType: 'application/json',
+        body: JSON.stringify({ id: 'test-id' }),
+      });
+    });
+
+    const url = await getLocalTestUrl({ testDir: __dirname });
+
+    await page.goto(url);
+    await reqPromise0;
+
+    await page.click('[data-console]');
+    await forceFlushReplay();
+
+    const { breadcrumbs } = getCustomRecordingEvents(await reqPromise1);
+
+    // 1 click breadcrumb + 1 throttled breadcrumb is why console logs are less
+    // than throttle limit
+    expect(breadcrumbs.length).toBe(THROTTLE_LIMIT);
+    expect(breadcrumbs.filter(breadcrumb => breadcrumb.category === 'replay.throttled').length).toBe(1);
+  },
+);
diff --git a/packages/browser-integration-tests/utils/replayHelpers.ts b/packages/browser-integration-tests/utils/replayHelpers.ts
@@ -34,6 +34,25 @@ export type IncrementalRecordingSnapshot = eventWithTime & {
 
 export type RecordingSnapshot = FullRecordingSnapshot | IncrementalRecordingSnapshot;
 
+/** Returns the replay event from the given request, or undefined if this is not a replay request. */
+export function getReplayEventFromRequest(req: Request): ReplayEvent | undefined {
+  const postData = req.postData();
+  if (!postData) {
+    return undefined;
+  }
+
+  try {
+    const event = envelopeRequestParser(req);
+
+    if (!isReplayEvent(event)) {
+      return undefined;
+    }
+
+    return event;
+  } catch {
+    return undefined;
+  }
+}
 /**
  * Waits for a replay request to be sent by the page and returns it.
  *
@@ -58,18 +77,13 @@ export function waitForReplayRequest(
     res => {
       const req = res.request();
 
-      const postData = req.postData();
-      if (!postData) {
+      const event = getReplayEventFromRequest(req);
+
+      if (!event) {
         return false;
       }
 
       try {
-        const event = envelopeRequestParser(req);
-
-        if (!isReplayEvent(event)) {
-          return false;
-        }
-
         if (callback) {
           return callback(event, res);
         }

diff --git a/packages/replay/.eslintrc.js b/packages/replay/.eslintrc.js
@@ -8,7 +8,9 @@ module.exports = {
   overrides: [
     {
       files: ['src/**/*.ts'],
-      rules: {},
+      rules: {
+        '@sentry-internal/sdk/no-unsupported-es6-methods': 'off',
+      },
     },
     {
       files: ['jest.setup.ts', 'jest.config.ts'],

diff --git a/packages/replay/src/coreHandlers/util/addBreadcrumbEvent.ts b/packages/replay/src/coreHandlers/util/addBreadcrumbEvent.ts
@@ -3,7 +3,6 @@ import type { Breadcrumb } from '@sentry/types';
 import { normalize } from '@sentry/utils';
 
 import type { ReplayContainer } from '../../types';
-import { addEvent } from '../../util/addEvent';
 
 /**
  * Add a breadcrumb event to replay.
@@ -20,7 +19,7 @@ export function addBreadcrumbEvent(replay: ReplayContainer, breadcrumb: Breadcru
   }
 
   replay.addUpdate(() => {
-    void addEvent(replay, {
+    void replay.throttledAddEvent({
       type: EventType.Custom,
       // TODO: We were converting from ms to seconds for breadcrumbs, spans,
       // but maybe we should just keep them as milliseconds

diff --git a/packages/replay/src/replay.ts b/packages/replay/src/replay.ts
@@ -24,6 +24,7 @@ import type {
   EventBuffer,
   InternalEventContext,
   PopEventContext,
+  RecordingEvent,
   RecordingOptions,
   ReplayContainer as ReplayContainerInterface,
   ReplayPluginOptions,
@@ -42,6 +43,8 @@ import { getHandleRecordingEmit } from './util/handleRecordingEmit';
 import { isExpired } from './util/isExpired';
 import { isSessionExpired } from './util/isSessionExpired';
 import { sendReplay } from './util/sendReplay';
+import type { SKIPPED } from './util/throttle';
+import { throttle, THROTTLED } from './util/throttle';
 
 /**
  * The main replay container class, which holds all the state and methods for recording and sending replays.
@@ -75,6 +78,11 @@ export class ReplayContainer implements ReplayContainerInterface {
     maxSessionLife: MAX_SESSION_LIFE,
   } as const;
 
+  private _throttledAddEvent: (
+    event: RecordingEvent,
+    isCheckout?: boolean,
+  ) => typeof THROTTLED | typeof SKIPPED | Promise<AddEventResult | null>;
+
   /**
    * Options to pass to `rrweb.record()`
    */
@@ -136,6 +144,14 @@ export class ReplayContainer implements ReplayContainerInterface {
     this._debouncedFlush = debounce(() => this._flush(), this._options.flushMinDelay, {
       maxWait: this._options.flushMaxDelay,
     });
+
+    this._throttledAddEvent = throttle(
+      (event: RecordingEvent, isCheckout?: boolean) => addEvent(this, event, isCheckout),
+      // Max 300 events...
+      300,
+      // ... per 5s
+      5,
+    );
   }
 
   /** Get the event context. */
@@ -565,6 +581,39 @@ export class ReplayContainer implements ReplayContainerInterface {
     this._context.urls.push(url);
   }
 
+  /**
+   * Add a breadcrumb event, that may be throttled.
+   * If it was throttled, we add a custom breadcrumb to indicate that.
+   */
+  public throttledAddEvent(
+    event: RecordingEvent,
+    isCheckout?: boolean,
+  ): typeof THROTTLED | typeof SKIPPED | Promise<AddEventResult | null> {
+    const res = this._throttledAddEvent(event, isCheckout);
+
+    // If this is THROTTLED, it means we have throttled the event for the first time
+    // In this case, we want to add a breadcrumb indicating that something was skipped
+    if (res === THROTTLED) {
+      const breadcrumb = createBreadcrumb({
+        category: 'replay.throttled',
+      });
+
+      this.addUpdate(() => {
+        void addEvent(this, {
+          type: EventType.Custom,
+          timestamp: breadcrumb.timestamp || 0,
+          data: {
+            tag: 'breadcrumb',
+            payload: breadcrumb,
+            metric: true,
+          },
+        });
+      });
+    }
+
+    return res;
+  }
+
   /**
    * Initialize and start all listeners to varying events (DOM,
    * Performance Observer, Recording, Sentry SDK, etc)
@@ -803,7 +852,7 @@ export class ReplayContainer implements ReplayContainerInterface {
    */
   private _createCustomBreadcrumb(breadcrumb: Breadcrumb): void {
     this.addUpdate(() => {
-      void addEvent(this, {
+      void this.throttledAddEvent({
         type: EventType.Custom,
         timestamp: breadcrumb.timestamp || 0,
         data: {

diff --git a/packages/replay/src/types.ts b/packages/replay/src/types.ts
@@ -8,6 +8,7 @@ import type {
 } from '@sentry/types';
 
 import type { eventWithTime, recordOptions } from './types/rrweb';
+import type { SKIPPED, THROTTLED } from './util/throttle';
 
 export type RecordingEvent = eventWithTime;
 export type RecordingOptions = recordOptions;
@@ -522,6 +523,10 @@ export interface ReplayContainer {
   session: Session | undefined;
   recordingMode: ReplayRecordingMode;
   timeouts: Timeouts;
+  throttledAddEvent: (
+    event: RecordingEvent,
+    isCheckout?: boolean,
+  ) => typeof THROTTLED | typeof SKIPPED | Promise<AddEventResult | null>;
   isEnabled(): boolean;
   isPaused(): boolean;
   getContext(): InternalEventContext;

diff --git a/packages/replay/src/util/createPerformanceSpans.ts b/packages/replay/src/util/createPerformanceSpans.ts
@@ -1,17 +1,16 @@
 import { EventType } from '@sentry-internal/rrweb';
 
 import type { AddEventResult, AllEntryData, ReplayContainer, ReplayPerformanceEntry } from '../types';
-import { addEvent } from './addEvent';
 
 /**
- * Create a "span" for each performance entry. The parent transaction is `this.replayEvent`.
+ * Create a "span" for each performance entry.
  */
 export function createPerformanceSpans(
   replay: ReplayContainer,
   entries: ReplayPerformanceEntry<AllEntryData>[],
 ): Promise<AddEventResult | null>[] {
-  return entries.map(({ type, start, end, name, data }) =>
-    addEvent(replay, {
+  return entries.map(({ type, start, end, name, data }) => {
+    const response = replay.throttledAddEvent({
       type: EventType.Custom,
       timestamp: start,
       data: {
@@ -24,6 +23,9 @@ export function createPerformanceSpans(
           data,
         },
       },
-    }),
-  );
+    });
+
+    // If response is a string, it means its either THROTTLED or SKIPPED
+    return typeof response === 'string' ? Promise.resolve(null) : response;
+  });
 }
diff --git a/packages/replay/src/util/throttle.ts b/packages/replay/src/util/throttle.ts
@@ -0,0 +1,55 @@
+export const THROTTLED = '__THROTTLED';
+export const SKIPPED = '__SKIPPED';
+
+/**
+ * Create a throttled function off a given function.
+ * When calling the throttled function, it will call the original function only
+ * if it hasn't been called more than `maxCount` times in the last `durationSeconds`.
+ *
+ * Returns `THROTTLED` if throttled for the first time, after that `SKIPPED`,
+ * or else the return value of the original function.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function throttle<T extends (...rest: any[]) => any>(
+  fn: T,
+  maxCount: number,
+  durationSeconds: number,
+): (...rest: Parameters<T>) => ReturnType<T> | typeof THROTTLED | typeof SKIPPED {
+  const counter = new Map<number, number>();
+
+  const _cleanup = (now: number): void => {
+    const threshold = now - durationSeconds;
+    counter.forEach((_value, key) => {
+      if (key < threshold) {
+        counter.delete(key);
+      }
+    });
+  };
+
+  const _getTotalCount = (): number => {
+    return [...counter.values()].reduce((a, b) => a + b, 0);
+  };
+
+  let isThrottled = false;
+
+  return (...rest: Parameters<T>): ReturnType<T> | typeof THROTTLED | typeof SKIPPED => {
+    // Date in second-precision, which we use as basis for the throttling
+    const now = Math.floor(Date.now() / 1000);
+
+    // First, make sure to delete any old entries
+    _cleanup(now);
+
+    // If already over limit, do nothing
+    if (_getTotalCount() >= maxCount) {
+      const wasThrottled = isThrottled;
+      isThrottled = true;
+      return wasThrottled ? SKIPPED : THROTTLED;
+    }
+
+    isThrottled = false;
+    const count = counter.get(now) || 0;
+    counter.set(now, count + 1);
+
+    return fn(...rest);
+  };
+}