fix: code

SocialGouv · May 14, 2024 · 68d1361 · 68d1361
1 parent 554d81a
commit 68d1361
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 29 deletions.
diff --git a/shared/elasticsearch/src/analysis.js b/shared/elasticsearch/src/analysis.js
@@ -66,51 +66,37 @@ const analyzer = {
   // for all suggestion
   // see below, ngram from tokens
   autocomplete: {
-    filter: ["lowercase", "icu_folding"],
+    filter: ["lowercase"],
     tokenizer: "autocomplete", //, "french_stop"]
   },
 
   // at search time, we only consider
   // the entire query (no ngrams)
   autocomplete_search: {
-    filter: "icu_folding",
     tokenizer: "lowercase",
   },
 
   french: {
-    filter: [
-      "french_elision",
-      "icu_folding",
-      "lowercase",
-      "french_stop",
-      "french_stemmer",
-    ],
-    tokenizer: "icu_tokenizer",
+    filter: ["french_elision", "lowercase", "french_stop", "french_stemmer"],
+    tokenizer: "standard",
   },
 
   french_indexing: {
     char_filter: ["startwith"],
-    filter: [
-      "french_elision",
-      "icu_folding",
-      "lowercase",
-      "french_stop",
-      "french_stemmer",
-    ],
-    tokenizer: "icu_tokenizer",
+    filter: ["french_elision", "lowercase", "french_stop", "french_stemmer"],
+    tokenizer: "standard",
   },
 
   french_with_synonyms: {
     char_filter: ["html_strip"],
     filter: [
       "french_elision",
-      "icu_folding",
       "lowercase",
       "french_synonyms",
       "french_stop",
       "french_stemmer",
     ],
-    tokenizer: "icu_tokenizer",
+    tokenizer: "standard",
   },
 
   idcc_ape: {
@@ -122,8 +108,8 @@ const analyzer = {
   // (as opposite to match "in the middle")
   sugg_prefix: {
     char_filter: ["startwith"],
-    filter: ["lowercase", "icu_folding"],
-    tokenizer: "icu_tokenizer",
+    filter: ["lowercase"],
+    tokenizer: "standard",
   },
 };
 

diff --git a/shared/elasticsearch/src/esClientUtils.js b/shared/elasticsearch/src/esClientUtils.js
@@ -3,7 +3,7 @@ const { logger } = require("@shared/utils");
 const { analyzer, char_filter, filter, tokenizer } = require("./analysis");
 
 async function createIndex({ client, indexName, mappings }) {
-  const { body } = await client.indices.exists({ index: indexName });
+  const body = await client.indices.exists({ index: indexName });
   if (body) {
     try {
       await client.indices.delete({ index: indexName });
@@ -38,7 +38,7 @@ async function createIndex({ client, indexName, mappings }) {
 }
 
 async function version({ client }) {
-  const { body } = await client.info();
+  const body = await client.info();
   logger.info(body.version.number);
 }
 
@@ -63,10 +63,8 @@ async function bulkIndexDocuments({ client, indexName, documents }) {
       ),
       index: indexName,
     });
-    if (resp.body.errors) {
-      const errorDocs = resp.body.items.filter(
-        (item) => item.index.status !== 201
-      );
+    if (resp.errors) {
+      const errorDocs = resp.items.filter((item) => item.index.status !== 201);
       throw new Error(`Error during indexing ${JSON.stringify(errorDocs)}`);
     }
     logger.info(`Index ${documents.length} documents.`);
@@ -89,7 +87,7 @@ async function indexDocumentsBatched({
 }
 
 async function deleteOldIndex({ client, patterns, timestamp }) {
-  const { body: indices } = await client.cat.indices({ format: "json" });
+  const indices = await client.cat.indices({ format: "json" });
 
   const IndicesToDelete = getIndicesToDelete(patterns, timestamp, indices);
   const pIndicesToDelete = IndicesToDelete.map(({ index }) =>

diff --git a/targets/export-elasticsearch/src/ingester/ingest.ts b/targets/export-elasticsearch/src/ingester/ingest.ts
@@ -151,12 +151,15 @@ async function runIngester(
         concurrency: 5,
       });
     }
+
+    console.log("ok");
     await indexDocumentsBatched({
       client,
       documents: docs,
       indexName: `${DOCUMENT_INDEX_NAME}-${ts}`,
       size: 800,
     });
+    console.log("mdr");
   };
   await cdtnDocumentsGen(updateDocs);