Merge pull request #1665 from quickwit-oss/fix_num_vals

fix num_vals on u128 value index after merge
quickwit-oss · Nov 7, 2022 · 3e9c806 · 3e9c806
2 parents 666afcf + c69a873
commit 3e9c806
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 6 deletions.
diff --git a/src/fastfield/multivalued/reader.rs b/src/fastfield/multivalued/reader.rs
@@ -183,6 +183,10 @@ impl<T: MonotonicallyMappableToU128> MultiValuedU128FastFieldReader<T> {
     /// Returns the overall number of values in this field.
     #[inline]
     pub fn total_num_vals(&self) -> u64 {
+        assert_eq!(
+            self.vals_reader.num_vals() as u64,
+            self.get_index_reader().total_num_vals()
+        );
         self.idx_reader.total_num_vals()
     }
 }

diff --git a/src/indexer/merger.rs b/src/indexer/merger.rs
@@ -356,6 +356,21 @@ impl IndexMerger {
                 .collect::<Vec<_>>(),
         )?;
 
+        let num_vals = segment_and_ff_readers
+            .iter()
+            .map(|(segment_reader, reader)| {
+                // TODO implement generic version, implement reverse scan, all - deletes
+                if let Some(alive_bitset) = segment_reader.alive_bitset() {
+                    alive_bitset
+                        .iter_alive()
+                        .map(|doc| reader.num_vals(doc))
+                        .sum()
+                } else {
+                    reader.total_num_vals() as u32
+                }
+            })
+            .sum();
+
         let fast_field_readers = segment_and_ff_readers
             .into_iter()
             .map(|(_, ff_reader)| ff_reader)
@@ -370,12 +385,7 @@ impl IndexMerger {
                 })
         };
 
-        fast_field_serializer.create_u128_fast_field_with_idx(
-            field,
-            iter_gen,
-            doc_id_mapping.len() as u32,
-            1,
-        )?;
+        fast_field_serializer.create_u128_fast_field_with_idx(field, iter_gen, num_vals, 1)?;
 
         Ok(())
     }