delete stale Algolia records after index update (#10058)

getsentry · May 16, 2024 · 5354049 · 5354049
1 parent a843665
commit 5354049
Showing 1 changed file with 44 additions and 15 deletions.
diff --git a/scripts/algolia.ts b/scripts/algolia.ts
@@ -1,3 +1,5 @@
+/* eslint-disable no-console */
+
 /**
  * This script is used to index the static docs HTML files generated by Next.js into Algolia.
  *
@@ -21,7 +23,7 @@ import fs from 'fs';
 import {join} from 'path';
 
 import {extrapolate, htmlToAlgoliaRecord} from '@sentry-internal/global-search';
-import algoliasearch from 'algoliasearch';
+import algoliasearch, {SearchIndex} from 'algoliasearch';
 
 import {getDocsFrontMatter} from '../src/mdx';
 import {FrontMatter} from '../src/types';
@@ -55,19 +57,47 @@ async function indexAndUpload() {
   // as they are used directly by generateStaticParams() on [[..path]] page
   const pageFrontMatters = await getDocsFrontMatter();
   const records = await generateAlogliaRecords(pageFrontMatters);
-  // eslint-disable-next-line no-console
-  console.log('🔥 Generated %d Algolia records.', records.length);
-  // eslint-disable-next-line no-console
-  console.log('🔥 Saving records ...');
-  await index
-    .saveObjects(records, {
-      batchSize: 10000,
-      autoGenerateObjectIDIfNotExist: true,
-    })
-    .then(result => {
-      // eslint-disable-next-line no-console
-      console.log('🔥 Saved %d Algolia records', result.objectIDs.length);
-    });
+  console.log('🔥 Generated %d new Algolia records.', records.length);
+  const existingRecordIds = await fetchExistingRecordIds(index);
+  console.log(
+    '🔥 Found %d existing Algolia records in `%s`',
+    existingRecordIds.length,
+    indexName
+  );
+  console.log('🔥 Saving new records to `%s`...', indexName);
+  const saveResult = await index.saveObjects(records, {
+    batchSize: 10000,
+    autoGenerateObjectIDIfNotExist: true,
+  });
+  const newRecordIDs = new Set(saveResult.objectIDs);
+  console.log('🔥 Saved %d records', newRecordIDs.size);
+
+  const recordsToDelete = existingRecordIds.filter(id => !newRecordIDs.has(id));
+  if (recordsToDelete.length === 0) {
+    console.log('🔥 No stale records to delete');
+    return;
+  }
+  console.log('🔥 Deleting old (stale) records ...');
+  const deleteResult = await index.deleteObjects(recordsToDelete);
+  console.log(
+    '🔥 Deleted %d stale records from `%s`',
+    deleteResult.objectIDs.length,
+    indexName
+  );
+}
+
+async function fetchExistingRecordIds(algoliaIndex: SearchIndex) {
+  console.log('🔥 fetching existing records ids ...');
+  const existingRecordIds = new Set<string>();
+  await algoliaIndex.browseObjects({
+    attributesToRetrieve: ['objectID'],
+    batch: chunk => {
+      chunk.forEach(record => {
+        existingRecordIds.add(record.objectID);
+      });
+    },
+  });
+  return Array.from(existingRecordIds);
 }
 
 async function generateAlogliaRecords(pageFrontMatters: FrontMatter[]) {
@@ -77,7 +107,6 @@ async function generateAlogliaRecords(pageFrontMatters: FrontMatter[]) {
         frontMatter => !frontMatter.draft && !frontMatter.noindex && frontMatter.title
       )
       .map(pageFm => {
-        // eslint-disable-next-line no-console
         console.log('processing:', pageFm.slug);
 
         const htmlFile = join(staticHtmlFilesPath, pageFm.slug + '.html');