log-server: reduce store's getLogs memory footprint (#220)

* llimit the number logs loaded in mem * test select query limit * changeset * fix missing logs with limitSelectQuery * fix lint
QuentinRoy · Jan 31, 2024 · 634945b · 634945b
1 parent 4cdf380
commit 634945b
Show file tree

Hide file tree

Showing 5 changed files with 411 additions and 182 deletions.
diff --git a/.changeset/old-zebras-tease.md b/.changeset/old-zebras-tease.md
@@ -0,0 +1,5 @@
+---
+'@lightmill/log-server': minor
+---
+
+Add a select query result limit for SQliteStore to prevent too many logs to be loaded in memory at the same time. Once the limit is attained logs are yielded and the next logs are loaded once their done being processed.
diff --git a/packages/log-server/__tests__/__snapshots__/sqlite-store.test.ts.snap b/packages/log-server/__tests__/__snapshots__/sqlite-store.test.ts.snap
@@ -0,0 +1,135 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`SQLiteStore#getLogs (selectQueryLimit: 2) > should return the logs in order of experimentId, runId, and ascending number 2`] = `
+[
+  {
+    "experimentId": "experiment1",
+    "number": 1,
+    "runId": "run1",
+    "type": "log1",
+    "values": {
+      "msg": "hello",
+      "recipient": "Anna",
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 2,
+    "runId": "run1",
+    "type": "log1",
+    "values": {
+      "msg": "bonjour",
+      "recipient": "Jo",
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 3,
+    "runId": "run1",
+    "type": "log3",
+    "values": {
+      "foo": true,
+      "x": 25,
+      "y": 0,
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 1,
+    "runId": "run2",
+    "type": "log1",
+    "values": {
+      "bar": null,
+      "message": "hola",
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 2,
+    "runId": "run2",
+    "type": "log2",
+    "values": {
+      "foo": false,
+      "x": 12,
+    },
+  },
+  {
+    "experimentId": "experiment2",
+    "number": 1,
+    "runId": "run1",
+    "type": "log2",
+    "values": {
+      "foo": true,
+      "x": 25,
+      "y": 0,
+    },
+  },
+]
+`;
+
+exports[`SQLiteStore#getLogs (selectQueryLimit: 10000) > should return the logs in order of experimentId, runId, and ascending number 2`] = `
+[
+  {
+    "experimentId": "experiment1",
+    "number": 1,
+    "runId": "run1",
+    "type": "log1",
+    "values": {
+      "msg": "hello",
+      "recipient": "Anna",
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 2,
+    "runId": "run1",
+    "type": "log1",
+    "values": {
+      "msg": "bonjour",
+      "recipient": "Jo",
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 3,
+    "runId": "run1",
+    "type": "log3",
+    "values": {
+      "foo": true,
+      "x": 25,
+      "y": 0,
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 1,
+    "runId": "run2",
+    "type": "log1",
+    "values": {
+      "bar": null,
+      "message": "hola",
+    },
+  },
+  {
+    "experimentId": "experiment1",
+    "number": 2,
+    "runId": "run2",
+    "type": "log2",
+    "values": {
+      "foo": false,
+      "x": 12,
+    },
+  },
+  {
+    "experimentId": "experiment2",
+    "number": 1,
+    "runId": "run1",
+    "type": "log2",
+    "values": {
+      "foo": true,
+      "x": 25,
+      "y": 0,
+    },
+  },
+]
+`;