diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index ef8821fbc4..689b82d87f 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -25,7 +25,7 @@ The Guava dependency version has been updated to 31.1. Projects may need to chec * **Performance** Looking up logical values from `DirectoryLayerDirectory`s no longer needs to create new transactions [(Issue #1857)](https://github.com/FoundationDB/fdb-record-layer/issues/1857) * **Performance** Improvement 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Performance** Improvement 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) -* **Feature** Feature 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) +* **Feature** Lucene search with highlighting the terms [(Issue #1862)](https://github.com/FoundationDB/fdb-record-layer/issues/1862) * **Feature** Feature 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) * **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/IndexEntry.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/IndexEntry.java index dc80725b4e..4e91007dc2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/IndexEntry.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/IndexEntry.java @@ -24,8 +24,10 @@ import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.Key.Evaluated.NullStandin; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.tuple.Tuple; import com.apple.foundationdb.tuple.TupleHelpers; +import com.google.protobuf.Message; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -228,6 +230,17 @@ public IndexEntry subKey(int startIdx, int endIdx) { return subKey; } + /** + * Rewrite the fetched stored record if needed. The default behavior is to keep the original fetched record. + * @param record the fetched record to rewrite + * @param type used to represent stored records + * @return the rewritten record + */ + @Nonnull + public FDBStoredRecord rewriteStoredRecord(@Nonnull FDBStoredRecord record) { + return record; + } + private void checkIfNullTypeAvailable() { // This indicates that the key/value was created from a tuple (i.e. likely values were read from an // index entry in the database) and, therefore, we don't know what type of null it was when it was diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java index feb7796263..43b418d855 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreBase.java @@ -1311,7 +1311,7 @@ default CompletableFuture> loadIndexEntryRecord(@Nonnull fin throw new RecordCoreException("Unexpected index orphan behavior: " + orphanBehavior); } } - return new FDBIndexedRecord<>(entry, rec); + return new FDBIndexedRecord<>(entry, entry.rewriteStoredRecord(rec)); }); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoredRecord.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoredRecord.java index af21340d17..d94bb7ba11 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoredRecord.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoredRecord.java @@ -150,6 +150,25 @@ public static FDBStoredRecordBuilder newBuilder(@Nonnull return new FDBStoredRecordBuilder<>(protoRecord); } + /** + * Get a builder with the parameters of a given {@link FDBStoredRecord} + * @param record given record + * @param type used to represent stored records + * @return a new builder + */ + @Nonnull + public static FDBStoredRecordBuilder newBuilder(@Nonnull FDBStoredRecord record) { + return new FDBStoredRecordBuilder<>(record.getRecord()) + .setPrimaryKey(record.getPrimaryKey()) + .setRecordType(record.getRecordType()) + .setKeyCount(record.getKeyCount()) + .setKeySize(record.getKeySize()) + .setValueSize(record.getValueSize()) + .setSplit(record.isSplit()) + .setVersion(record.getVersion()) + .setVersionedInline(record.isVersionedInline()); + } + /** * Copy this record with a different version. * @param recordVersion new version diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursor.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursor.java index 317e1d635d..894bea2af1 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursor.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursor.java @@ -78,6 +78,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -192,8 +193,10 @@ private void performLookup() throws IOException { @SuppressWarnings("squid:S3776") // Cognitive complexity is too high. Candidate for later refactoring @Nullable @VisibleForTesting - static String searchAllMaybeHighlight(Analyzer queryAnalyzer, String text, Set matchedTokens, @Nullable String prefixToken, boolean highlight) { - try (TokenStream ts = queryAnalyzer.tokenStream("text", new StringReader(text))) { + static String searchAllMaybeHighlight(@Nonnull String fieldName, @Nonnull Analyzer queryAnalyzer, @Nonnull String text, + @Nonnull Set matchedTokens, @Nullable String prefixToken, + boolean highlight, boolean allMatchingRequired) { + try (TokenStream ts = queryAnalyzer.tokenStream(fieldName, new StringReader(text))) { CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); ts.reset(); @@ -217,13 +220,30 @@ static String searchAllMaybeHighlight(Analyzer queryAnalyzer, String text, Set= 0 + && endOffset + 4 > text.length() + && text.startsWith("", startOffset - 3) + && text.startsWith("", endOffset); + } + /** Called while highlighting a single result, to append a * non-matching chunk of text from the suggestion to the * provided fragments list. @@ -532,7 +560,7 @@ private RecordCursor findIndexEntriesInRecord(ScoreDocAndRecord scor // matched terms return null; } - String match = searchAllMaybeHighlight(queryAnalyzer, text, queryTokens, prefixToken, highlight); + String match = searchAllMaybeHighlight(documentField.getFieldName(), queryAnalyzer, text, queryTokens, prefixToken, highlight, true); if (match == null) { // Text not found in this field return null; diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecord.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecord.java index 97b71da39b..325f47483e 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecord.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecord.java @@ -38,9 +38,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; /** * Helper class for converting {@link FDBRecord}s to Lucene documents. @@ -131,6 +134,41 @@ public static List getFields(@Nonnull KeyExpr return fields.getFields(); } + // Modify the Lucene fields of a record message with highlighting the terms from the given termMap + @Nonnull + public static void highlightTermsInMessage(@Nonnull KeyExpression expression, @Nonnull Message.Builder builder, @Nonnull Map> termMap, + @Nonnull LuceneAnalyzerCombinationProvider analyzerSelector) { + LuceneIndexKeyValueToPartialRecordUtils.RecordRebuildSource recordRebuildSource = new LuceneIndexKeyValueToPartialRecordUtils.RecordRebuildSource<>(null, builder.getDescriptorForType(), builder, builder.build()); + + LuceneIndexExpressions.getFields(expression, recordRebuildSource, + (source, fieldName, value, type, stored, sorted, overriddenKeyRanges, groupingKeyIndex, keyIndex, fieldConfigsIgnored) -> { + Set terms = new HashSet<>(); + terms.addAll(termMap.getOrDefault(fieldName, Collections.emptySet())); + terms.addAll(termMap.getOrDefault("", Collections.emptySet())); + if (terms.isEmpty()) { + return; + } + for (Map.Entry entry : source.message.getAllFields().entrySet()) { + Object entryValue = entry.getValue(); + if (entryValue instanceof String && entryValue.equals(value) + && terms.stream().filter(t -> ((String) entryValue).toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT))).findAny().isPresent()) { + String highlightedText = LuceneAutoCompleteResultCursor.searchAllMaybeHighlight(fieldName, analyzerSelector.provideIndexAnalyzer((String) entryValue).getAnalyzer(), (String) entryValue, termMap.get(fieldName), null, true, false); + source.buildMessage(highlightedText, entry.getKey(), null, null, true, 0); + } else if (entryValue instanceof List) { + int index = 0; + for (Object entryValueElement : ((List) entryValue)) { + if (entryValueElement instanceof String && entryValueElement.equals(value) + && terms.stream().filter(t -> ((String) entryValueElement).toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT))).findAny().isPresent()) { + String highlightedText = LuceneAutoCompleteResultCursor.searchAllMaybeHighlight(fieldName, analyzerSelector.provideIndexAnalyzer((String) entryValueElement).getAnalyzer(), (String) entryValueElement, termMap.get(fieldName), null, true, false); + source.buildMessage(highlightedText, entry.getKey(), null, null, true, index); + } + index++; + } + } + } + }, null); + } + protected static class FDBRecordSource implements LuceneIndexExpressions.RecordSource> { @Nonnull private final FDBRecord rec; diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexKeyValueToPartialRecordUtils.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexKeyValueToPartialRecordUtils.java index 63c1ada4c2..20e512b6e8 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexKeyValueToPartialRecordUtils.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexKeyValueToPartialRecordUtils.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.IndexEntry; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -200,6 +201,115 @@ private static Pair, List> getOriginalAndMappedFieldElement return Pair.of(fixedFieldNames, dynamicFieldNames); } + static class RecordRebuildSource implements LuceneIndexExpressions.RecordSource> { + @Nullable + public final RecordRebuildSource parent; + @Nonnull + public final Descriptors.Descriptor descriptor; + @Nullable + public final Descriptors.FieldDescriptor fieldDescriptor; + @Nonnull + public final Message.Builder builder; + public final Message message; + public final int indexIfRepeated; + + RecordRebuildSource(@Nullable RecordRebuildSource parent, @Nonnull Descriptors.Descriptor descriptor, @Nonnull Message.Builder builder, @Nonnull Message message) { + //this.rec = rec; + this.parent = parent; + this.descriptor = descriptor; + this.fieldDescriptor = null; + this.builder = builder; + this.message = message; + this.indexIfRepeated = 0; + } + + RecordRebuildSource(@Nullable RecordRebuildSource parent, @Nonnull Descriptors.FieldDescriptor fieldDescriptor, @Nonnull Message.Builder builder, @Nonnull Message message, int indexIfRepeated) { + //this.rec = rec; + this.parent = parent; + this.descriptor = fieldDescriptor.getMessageType(); + this.fieldDescriptor = fieldDescriptor; + this.builder = builder; + this.message = message; + this.indexIfRepeated = indexIfRepeated; + } + + @Override + public Descriptors.Descriptor getDescriptor() { + return descriptor; + } + + @Override + public Iterable> getChildren(@Nonnull FieldKeyExpression parentExpression) { + final String parentField = parentExpression.getFieldName(); + final Descriptors.FieldDescriptor parentFieldDescriptor = descriptor.findFieldByName(parentField); + + final List> children = new ArrayList<>(); + int index = 0; + for (Key.Evaluated evaluated : parentExpression.evaluateMessage(null, message)) { + final Message submessage = (Message)evaluated.toList().get(0); + if (submessage != null) { + if (parentFieldDescriptor.isRepeated()) { + children.add(new RecordRebuildSource(this, parentFieldDescriptor, + builder.newBuilderForField(parentFieldDescriptor), + submessage, index++)); + } else { + children.add(new RecordRebuildSource(this, parentFieldDescriptor, + builder.getFieldBuilder(parentFieldDescriptor), + submessage, index)); + } + } + } + return children; + } + + @Override + public Iterable getValues(@Nonnull FieldKeyExpression fieldExpression) { + final List values = new ArrayList<>(); + for (Key.Evaluated evaluated : fieldExpression.evaluateMessage(null, message)) { + Object value = evaluated.getObject(0); + if (value != null) { + values.add(value); + } + } + return values; + } + + @SuppressWarnings("java:S3776") + public void buildMessage(@Nullable Object value, Descriptors.FieldDescriptor subFieldDescriptor, @Nullable String customizedKey, @Nullable String mappedKeyField, boolean forLuceneField, int index) { + final Descriptors.FieldDescriptor mappedKeyFieldDescriptor = mappedKeyField == null ? null : descriptor.findFieldByName(mappedKeyField); + if (mappedKeyFieldDescriptor != null) { + if (customizedKey == null) { + return; + } + builder.setField(mappedKeyFieldDescriptor, customizedKey); + } + + if (value == null) { + return; + } + if (subFieldDescriptor.isRepeated()) { + if (subFieldDescriptor.getJavaType().equals(Descriptors.FieldDescriptor.JavaType.MESSAGE)) { + Message.Builder subBuilder = builder.newBuilderForField(subFieldDescriptor); + subBuilder.mergeFrom((Message) builder.getRepeatedField(subFieldDescriptor, index)).mergeFrom((Message) value); + builder.setRepeatedField(subFieldDescriptor, index, subBuilder.build()); + } else { + builder.setRepeatedField(subFieldDescriptor, index, value); + } + + } else { + int count = builder.getAllFields().size(); + if (message != null && count == 0) { + builder.mergeFrom(message); + } + builder.setField(subFieldDescriptor, value); + } + + if (parent != null) { + parent.buildMessage(builder.build(), this.fieldDescriptor, mappedKeyFieldDescriptor == null ? customizedKey : null, mappedKeyFieldDescriptor == null ? mappedKeyField : null, forLuceneField, indexIfRepeated); + } + } + } + /** * A {@link com.apple.foundationdb.record.lucene.LuceneIndexExpressions.RecordSource} implementation to build the partial record message. */ diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java index 510bdf2f20..8749175c42 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java @@ -133,7 +133,8 @@ public RecordCursor scan(@Nonnull final IndexScanBounds scanBounds, return new LuceneRecordCursor(executor, state.context.getPropertyStorage().getPropertyValue(LuceneRecordContextProperties.LUCENE_EXECUTOR_SERVICE), state.context.getPropertyStorage().getPropertyValue(LuceneRecordContextProperties.LUCENE_INDEX_CURSOR_PAGE_SIZE), scanProperties, state, scanQuery.getQuery(), scanQuery.getSort(), continuation, - scanQuery.getGroupKey(), scanQuery.getStoredFields(), scanQuery.getStoredFieldTypes()); + scanQuery.getGroupKey(), scanQuery.isHighlight(), + scanQuery.getStoredFields(), scanQuery.getStoredFieldTypes(), indexAnalyzerSelector); } if (scanType.equals(LuceneScanTypes.BY_LUCENE_AUTO_COMPLETE)) { diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LucenePlanner.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LucenePlanner.java index 2c1e616c8d..a56530c14e 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LucenePlanner.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LucenePlanner.java @@ -120,8 +120,9 @@ private ScoredPlan planLucene(@Nonnull CandidateScan candidateScan, LucenePlanState state = new LucenePlanState(index, groupingComparisons, filter); getFieldDerivations(state); + QueryComponent queryComponent = state.groupingComparisons.isEmpty() ? state.filter : filterMask.getUnsatisfiedFilter(); // Special scans like auto-complete cannot be combined with regular queries. - LuceneScanParameters scanParameters = getSpecialScan(state, filterMask); + LuceneScanParameters scanParameters = getSpecialScan(state, filterMask, queryComponent); if (scanParameters == null) { // Scan by means of normal Lucene search API. LuceneQueryClause query = getQueryForFilter(state, filter, new ArrayList<>(), filterMask); @@ -133,7 +134,7 @@ private ScoredPlan planLucene(@Nonnull CandidateScan candidateScan, } getStoredFields(state); scanParameters = new LuceneScanQueryParameters(groupingComparisons, query, - state.sort, state.storedFields, state.storedFieldTypes); + state.sort, state.storedFields, state.storedFieldTypes, toHighLightOrNot(queryComponent)); } // Wrap in plan. @@ -148,6 +149,25 @@ private ScoredPlan planLucene(@Nonnull CandidateScan candidateScan, state.repeated, null); } + private static boolean toHighLightOrNot(@Nonnull QueryComponent queryComponent) { + if (queryComponent instanceof LuceneQueryComponent) { + return ((LuceneQueryComponent) queryComponent).getType().equals(LuceneQueryComponent.Type.QUERY_HIGHLIGHT); + } else if (queryComponent instanceof AndOrComponent) { + for (QueryComponent child : ((AndOrComponent) queryComponent).getChildren()) { + if (toHighLightOrNot((child))) { + return true; + } + } + } else if (queryComponent instanceof AndComponent) { + for (QueryComponent child : ((AndComponent) queryComponent).getChildren()) { + if (toHighLightOrNot(child)) { + return true; + } + } + } + return false; + } + static class LucenePlanState { @Nonnull final Index index; @@ -178,8 +198,7 @@ static class LucenePlanState { @Nullable @SuppressWarnings("PMD.CompareObjectsWithEquals") - private LuceneScanParameters getSpecialScan(@Nonnull LucenePlanState state, @Nonnull FilterSatisfiedMask filterMask) { - QueryComponent queryComponent = state.groupingComparisons.isEmpty() ? state.filter : filterMask.getUnsatisfiedFilter(); + private LuceneScanParameters getSpecialScan(@Nonnull LucenePlanState state, @Nonnull FilterSatisfiedMask filterMask, @Nonnull QueryComponent queryComponent) { if (queryComponent instanceof LuceneQueryComponent) { LuceneQueryComponent luceneQueryComponent = (LuceneQueryComponent)queryComponent; for (String field : luceneQueryComponent.getFields()) { diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneQueryComponent.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneQueryComponent.java index be380923de..3b2faa11c3 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneQueryComponent.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneQueryComponent.java @@ -54,6 +54,7 @@ public class LuceneQueryComponent implements QueryComponent, ComponentWithNoChil */ public enum Type { QUERY, + QUERY_HIGHLIGHT, AUTO_COMPLETE_HIGHLIGHT, AUTO_COMPLETE, SPELL_CHECK, diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneRecordCursor.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneRecordCursor.java index b8b0ce754b..87258fd595 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneRecordCursor.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneRecordCursor.java @@ -38,18 +38,27 @@ import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerState; import com.apple.foundationdb.tuple.Tuple; import com.apple.foundationdb.tuple.TupleHelpers; import com.google.common.collect.Lists; +import com.google.protobuf.Message; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexNotFoundException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -60,9 +69,14 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -114,6 +128,10 @@ class LuceneRecordCursor implements BaseCursor { @Nullable private final List storedFieldTypes; + private final boolean highlight; + @Nonnull + private final LuceneAnalyzerCombinationProvider analyzerSelector; + //TODO: once we fix the available fields logic for lucene to take into account which fields are // stored there should be no need to pass in a list of fields, or we could only pass in the store field values. @SuppressWarnings("squid:S107") @@ -126,8 +144,10 @@ class LuceneRecordCursor implements BaseCursor { @Nullable Sort sort, byte[] continuation, @Nullable Tuple groupingKey, + boolean highlight, @Nullable final List storedFields, - @Nullable final List storedFieldTypes) { + @Nullable final List storedFieldTypes, + @Nonnull LuceneAnalyzerCombinationProvider analyzerSelector) { this.state = state; this.executor = executor; this.pageSize = pageSize; @@ -151,6 +171,8 @@ class LuceneRecordCursor implements BaseCursor { } this.fields = state.index.getRootExpression().normalizeKeyForPositions(); this.groupingKey = groupingKey; + this.highlight = highlight; + this.analyzerSelector = analyzerSelector; } @Nonnull @@ -351,23 +373,84 @@ private CompletableFuture buildIndexEntryFromScoreDocAsync(@ tuple = Tuple.fromList(fieldValues).addAll(setPrimaryKey); } - return new ScoreDocIndexEntry(scoreDoc, state.index, tuple); + return new ScoreDocIndexEntry(scoreDoc, state.index, tuple, highlight, query, analyzerSelector); } catch (Exception e) { throw new RecordCoreException("Failed to get document", "currentPosition", currentPosition, "exception", e); } }, executor); } + // Parse the Lucene query to get all the mapping from field to terms + private static void getTerms(Query query, Map> map) { + if (query instanceof BooleanQuery) { + BooleanQuery booleanQuery = (BooleanQuery) query; + for (BooleanClause clause : booleanQuery.clauses()) { + getTerms(clause.getQuery(), map); + } + } else if (query instanceof TermQuery) { + TermQuery termQuery = (TermQuery) query; + Term term = termQuery.getTerm(); + map.putIfAbsent(term.field(), new HashSet<>()); + map.get(term.field()).add(term.text().toLowerCase(Locale.ROOT)); + } else if (query instanceof PhraseQuery) { + PhraseQuery phraseQuery = (PhraseQuery) query; + for (Term term : phraseQuery.getTerms()) { + map.putIfAbsent(term.field(), new HashSet<>()); + map.get(term.field()).add(term.text().toLowerCase(Locale.ROOT)); + } + } else if (query instanceof MultiPhraseQuery) { + MultiPhraseQuery multiPhraseQuery = (MultiPhraseQuery) query; + for (Term[] termArray : multiPhraseQuery.getTermArrays()) { + for (Term term : termArray) { + map.putIfAbsent(term.field(), new HashSet<>()); + map.get(term.field()).add(term.text().toLowerCase(Locale.ROOT)); + } + } + } else if (query instanceof BoostQuery) { + BoostQuery boostQuery = (BoostQuery) query; + getTerms(boostQuery.getQuery(), map); + } else { + throw new RecordCoreException("This lucene query is not supported for highlighting"); + } + } + protected static final class ScoreDocIndexEntry extends IndexEntry { private final ScoreDoc scoreDoc; + private final Map> termMap; + private final LuceneAnalyzerCombinationProvider analyzerSelector; + private final boolean highlight; + private final KeyExpression indexKey; + public ScoreDoc getScoreDoc() { return scoreDoc; } - private ScoreDocIndexEntry(@Nonnull ScoreDoc scoreDoc, @Nonnull Index index, @Nonnull Tuple key) { + private ScoreDocIndexEntry(@Nonnull ScoreDoc scoreDoc, @Nonnull Index index, @Nonnull Tuple key, + boolean highlight, @Nonnull Query query, + @Nonnull LuceneAnalyzerCombinationProvider analyzerSelector) { super(index, key, TupleHelpers.EMPTY); this.scoreDoc = scoreDoc; + this.highlight = highlight; + this.termMap = new HashMap<>(); + this.analyzerSelector = analyzerSelector; + this.indexKey = index.getRootExpression(); + if (highlight) { + getTerms(query, this.termMap); + } + } + + @SuppressWarnings("unchecked") + @Override + @Nonnull + public FDBStoredRecord rewriteStoredRecord(@Nonnull FDBStoredRecord record) { + if (!highlight) { + return super.rewriteStoredRecord(record); + } + M message = record.getRecord(); + M.Builder builder = message.toBuilder(); + LuceneDocumentFromRecord.highlightTermsInMessage(indexKey, builder, termMap, analyzerSelector); + return FDBStoredRecord.newBuilder(record).setRecord((M) builder.build()).build(); } @Override diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQuery.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQuery.java index 7b4faaf8df..8ee835dc72 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQuery.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQuery.java @@ -44,14 +44,18 @@ public class LuceneScanQuery extends LuceneScanBounds { @Nullable private final List storedFieldTypes; + private final boolean highlight; + public LuceneScanQuery(@Nonnull IndexScanType scanType, @Nonnull Tuple groupKey, - @Nonnull Query query, @Nullable Sort sort, - @Nullable List storedFields, @Nullable List storedFieldTypes) { + @Nonnull Query query, @Nullable Sort sort, @Nullable List storedFields, + @Nullable List storedFieldTypes, + boolean highlight) { super(scanType, groupKey); this.query = query; this.sort = sort; this.storedFields = storedFields; this.storedFieldTypes = storedFieldTypes; + this.highlight = highlight; } @Nonnull @@ -74,6 +78,10 @@ public List getStoredFieldTypes() { return storedFieldTypes; } + public boolean isHighlight() { + return highlight; + } + @Override public String toString() { return super.toString() + " " + query; diff --git a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQueryParameters.java b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQueryParameters.java index 13d3bcf0f7..48c334319f 100644 --- a/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQueryParameters.java +++ b/fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneScanQueryParameters.java @@ -60,18 +60,22 @@ public class LuceneScanQueryParameters extends LuceneScanParameters { @Nullable final List storedFieldTypes; + final boolean highlight; + public LuceneScanQueryParameters(@Nonnull ScanComparisons groupComparisons, @Nonnull LuceneQueryClause query) { - this(groupComparisons, query, null, null, null); + this(groupComparisons, query, null, null, null, false); } public LuceneScanQueryParameters(@Nonnull ScanComparisons groupComparisons, @Nonnull LuceneQueryClause query, @Nullable Sort sort, - @Nullable List storedFields, @Nullable List storedFieldTypes) { + @Nullable List storedFields, @Nullable List storedFieldTypes, + boolean highlight) { super(LuceneScanTypes.BY_LUCENE, groupComparisons); this.query = query; this.sort = sort; this.storedFields = storedFields; this.storedFieldTypes = storedFieldTypes; + this.highlight = highlight; } @Nonnull @@ -103,7 +107,7 @@ public int planHash(@Nonnull PlanHashKind hashKind) { @Override public LuceneScanQuery bind(@Nonnull FDBRecordStoreBase store, @Nonnull Index index, @Nonnull EvaluationContext context) { return new LuceneScanQuery(scanType, getGroupKey(store, context), query.bind(store, index, context), - sort, storedFields, storedFieldTypes); + sort, storedFields, storedFieldTypes, highlight); } @Nonnull diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java index 80c568d8aa..758a1426f0 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/FDBLuceneQueryTest.java @@ -38,8 +38,10 @@ import com.apple.foundationdb.record.provider.common.text.AllSuffixesTextTokenizer; import com.apple.foundationdb.record.provider.common.text.TextSamples; import com.apple.foundationdb.record.provider.foundationdb.FDBQueriedRecord; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContextConfig; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils; import com.apple.foundationdb.record.provider.foundationdb.properties.RecordLayerPropertyStorage; import com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase; @@ -61,7 +63,11 @@ import org.hamcrest.Matchers; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; import org.junit.jupiter.params.provider.MethodSource; import javax.annotation.Nonnull; @@ -113,6 +119,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasToString; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Sophisticated queries involving full text predicates. @@ -120,6 +127,16 @@ @Tag(Tags.RequiresFDB) public class FDBLuceneQueryTest extends FDBRecordStoreQueryTestBase { + static class BooleanArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of(Arguments.of(false, false), + Arguments.of(true, false), + Arguments.of(false, true), + Arguments.of(true, true)); + } + } + private static final List DOCUMENTS = TextIndexTestUtils.toSimpleDocuments(Arrays.asList( TextSamples.ANGSTROM, TextSamples.AETHELRED, @@ -392,13 +409,14 @@ void testNgramEdgesOnly(boolean shouldDeferFetch) throws Exception { assertTermIndexedOrNot("rning", false, shouldDeferFetch); } - @ParameterizedTest(name = "simpleLuceneScans[shouldDeferFetch={0}]") - @BooleanSource - void simpleLuceneScans(boolean shouldDeferFetch) throws Exception { + @ParameterizedTest + @ArgumentsSource(BooleanArgumentsProvider.class) + void simpleLuceneScans(boolean shouldDeferFetch, boolean withHighlight) throws Exception { initializeFlat(); try (FDBRecordContext context = openContext()) { openRecordStore(context); - final QueryComponent filter1 = new LuceneQueryComponent("civil blood makes civil hands unclean", Lists.newArrayList()); + final QueryComponent filter1 = new LuceneQueryComponent(withHighlight ? LuceneQueryComponent.Type.QUERY_HIGHLIGHT : LuceneQueryComponent.Type.QUERY, + "civil blood makes civil hands unclean", false, Lists.newArrayList(), true); // Query for full records RecordQuery query = RecordQuery.newBuilder() .setRecordType(TextIndexTestUtils.SIMPLE_DOC) @@ -410,10 +428,12 @@ void simpleLuceneScans(boolean shouldDeferFetch) throws Exception { scanParams(query(hasToString("MULTI civil blood makes civil hands unclean"))))); RecordQueryPlan plan = planner.plan(query); assertThat(plan, matcher); - RecordCursor> fdbQueriedRecordRecordCursor = recordStore.executeQuery(plan); - RecordCursor map = fdbQueriedRecordRecordCursor.map(FDBQueriedRecord::getPrimaryKey); - List primaryKeys = map.map(t -> t.getLong(0)).asList().get(); + List> queriedRecordList = recordStore.executeQuery(plan).asList().get(); + Set primaryKeys = queriedRecordList.stream().map(FDBQueriedRecord::getPrimaryKey).map(t -> t.getLong(0)).collect(Collectors.toSet()); assertEquals(Set.of(2L, 4L), Set.copyOf(primaryKeys)); + + Set texts = queriedRecordList.stream().map(FDBQueriedRecord::getStoredRecord).map(FDBStoredRecord::getRecord).map(m -> (String) m.getField(m.getDescriptorForType().findFieldByName("text"))).collect(Collectors.toSet()); + texts.forEach(t -> assertTrue(t.contains(withHighlight ? "civil blood makes civil hands unclean" : "civil blood makes civil hands unclean"))); } } @@ -546,14 +566,14 @@ void delayFetchOnAndOfLuceneAndFieldFilter(boolean shouldDeferFetch) throws Exce } } - @ParameterizedTest(name = "delayFetchOnOrOfLuceneFiltersGivesUnion[shouldDeferFetch={0}]") - @BooleanSource - void delayFetchOnOrOfLuceneFiltersGivesUnion(boolean shouldDeferFetch) throws Exception { + @ParameterizedTest + @ArgumentsSource(BooleanArgumentsProvider.class) + void delayFetchOnOrOfLuceneFiltersGivesUnion(boolean shouldDeferFetch, boolean withHighlight) throws Exception { initializeFlat(); try (FDBRecordContext context = openContext()) { openRecordStore(context); - final QueryComponent filter1 = new LuceneQueryComponent("(\"civil blood makes civil hands unclean\")", Lists.newArrayList("text"), true); - final QueryComponent filter2 = new LuceneQueryComponent("(\"was king from 966 to 1016\")", Lists.newArrayList()); + final QueryComponent filter1 = new LuceneQueryComponent(withHighlight ? LuceneQueryComponent.Type.QUERY_HIGHLIGHT : LuceneQueryComponent.Type.QUERY, "(\"civil blood makes civil hands unclean\")", false, Lists.newArrayList("text"), true); + final QueryComponent filter2 = new LuceneQueryComponent(withHighlight ? LuceneQueryComponent.Type.QUERY_HIGHLIGHT : LuceneQueryComponent.Type.QUERY, "(\"was king from 966 to 1016\")", false, Lists.newArrayList(), true); // Query for full records RecordQuery query = RecordQuery.newBuilder() .setRecordType(TextIndexTestUtils.SIMPLE_DOC) @@ -578,24 +598,34 @@ void delayFetchOnOrOfLuceneFiltersGivesUnion(boolean shouldDeferFetch) throws Ex scanParams(query(hasToString("MULTI (\"was king from 966 to 1016\")"))))))))); } assertThat(plan, matcher); - List primaryKeys = recordStore.executeQuery(plan).map(FDBQueriedRecord::getPrimaryKey).map(t -> t.getLong(0)).asList().get(); - assertEquals(Set.of(1L, 2L, 4L), Set.copyOf(primaryKeys)); + List> queriedRecordList = recordStore.executeQuery(plan).asList().get(); + Set primaryKeys = queriedRecordList.stream().map(FDBRecord::getPrimaryKey).map(t -> t.getLong(0)).collect(Collectors.toSet()); + assertEquals(Set.of(1L, 2L, 4L), primaryKeys); if (shouldDeferFetch) { assertLoadRecord(5, context); } else { assertLoadRecord(6, context); } + + Set texts = queriedRecordList.stream().map(FDBQueriedRecord::getStoredRecord).map(FDBStoredRecord::getRecord).map(m -> (String) m.getField(m.getDescriptorForType().findFieldByName("text"))).collect(Collectors.toSet()); + for (String text : texts) { + boolean match1 = text.contains(withHighlight ? "was king from 966 to 1016" : "was king from 966 to 1016"); + boolean match2 = text.contains(withHighlight ? "civil blood makes civil hands unclean" : "civil blood makes civil hands unclean"); + assertTrue(match1 || match2); + } } } - @ParameterizedTest(name = "delayFetchOnAndOfLuceneFilters[shouldDeferFetch={0}]") - @BooleanSource - void delayFetchOnAndOfLuceneFilters(boolean shouldDeferFetch) throws Exception { + @ParameterizedTest + @ArgumentsSource(BooleanArgumentsProvider.class) + void delayFetchOnAndOfLuceneFilters(boolean shouldDeferFetch, boolean withHighlight) throws Exception { initializeFlat(); try (FDBRecordContext context = openContext()) { openRecordStore(context); - final QueryComponent filter1 = new LuceneQueryComponent("\"the continuance\"", Lists.newArrayList()); - final QueryComponent filter2 = new LuceneQueryComponent("grudge", Lists.newArrayList()); + final QueryComponent filter1 = new LuceneQueryComponent(withHighlight ? LuceneQueryComponent.Type.QUERY_HIGHLIGHT : LuceneQueryComponent.Type.QUERY, + "\"the continuance\"", false, Lists.newArrayList(), true); + final QueryComponent filter2 = new LuceneQueryComponent(withHighlight ? LuceneQueryComponent.Type.QUERY_HIGHLIGHT : LuceneQueryComponent.Type.QUERY, + "grudge", false, Lists.newArrayList(), true); // Query for full records RecordQuery query = RecordQuery.newBuilder() .setRecordType(TextIndexTestUtils.SIMPLE_DOC) @@ -607,13 +637,21 @@ void delayFetchOnAndOfLuceneFilters(boolean shouldDeferFetch) throws Exception { indexName(SIMPLE_TEXT_SUFFIXES.getName()), scanParams(query(hasToString("MULTI \"the continuance\" AND MULTI grudge"))))); assertThat(plan, matcher); - List primaryKeys = recordStore.executeQuery(plan).map(FDBQueriedRecord::getPrimaryKey).map(t -> t.getLong(0)).asList().get(); - assertEquals(Set.of(4L), Set.copyOf(primaryKeys)); + List> queriedRecordList = recordStore.executeQuery(plan).asList().get(); + Set primaryKeys = queriedRecordList.stream().map(FDBQueriedRecord::getPrimaryKey).map(t -> t.getLong(0)).collect(Collectors.toSet()); + assertEquals(Set.of(4L), primaryKeys); if (shouldDeferFetch) { assertLoadRecord(3, context); } else { assertLoadRecord(4, context); } + + Set texts = queriedRecordList.stream().map(FDBQueriedRecord::getStoredRecord).map(FDBStoredRecord::getRecord).map(m -> (String) m.getField(m.getDescriptorForType().findFieldByName("text"))).collect(Collectors.toSet()); + for (String text : texts) { + boolean match1 = text.contains(withHighlight ? "the continuance" : "the continuance"); + boolean match2 = text.contains(withHighlight ? "grudge" : "grudge"); + assertTrue(match1 || match2); + } } } diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursorTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursorTest.java index 41bd16331d..9a0e06b3e9 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursorTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneAutoCompleteResultCursorTest.java @@ -126,7 +126,7 @@ private static void assertSearchMatches(String queryString, List expecte assertEquals(expectedPrefixToken, prefixToken); Set queryTokenSet = new HashSet<>(tokens); - @Nullable String match = LuceneAutoCompleteResultCursor.searchAllMaybeHighlight(analyzer, text, queryTokenSet, prefixToken, highlight); + @Nullable String match = LuceneAutoCompleteResultCursor.searchAllMaybeHighlight("text", analyzer, text, queryTokenSet, prefixToken, highlight, true); assertEquals(expectedMatch, match); } } diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecordTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecordTest.java index 94a1acfaf2..5389b31381 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecordTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecordTest.java @@ -34,9 +34,9 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; - import java.util.Collections; import java.util.Map; +import java.util.Set; import static com.apple.foundationdb.record.metadata.Key.Expressions.concat; import static com.apple.foundationdb.record.metadata.Key.Expressions.field; @@ -50,25 +50,33 @@ */ class LuceneDocumentFromRecordTest { + private LuceneAnalyzerCombinationProvider analyzerProvider = new LuceneAnalyzerCombinationProvider(t -> LuceneAnalyzerWrapper.getStandardAnalyzerWrapper(), + t -> LuceneAnalyzerWrapper.getStandardAnalyzerWrapper(), + null, null); + @Test void simple() { - TestRecordsTextProto.SimpleDocument message = TestRecordsTextProto.SimpleDocument.newBuilder() + TestRecordsTextProto.SimpleDocument.Builder builder = TestRecordsTextProto.SimpleDocument.newBuilder() .setDocId(1) - .setText("some text") - .build(); + .setText("some text"); + TestRecordsTextProto.SimpleDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = function(LuceneFunctionNames.LUCENE_TEXT, field("text")); assertEquals(ImmutableMap.of(Tuple.from(), ImmutableList.of(textField("text", "some text"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "some" for text field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("text", Set.of("some")), analyzerProvider); + assertEquals("some text", builder.build().getText()); + KeyExpression primaryKey = field("doc_id"); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.SimpleDocument.Builder builder = TestRecordsTextProto.SimpleDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "text", "suggestion"); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(1)); - TestRecordsTextProto.SimpleDocument partialMsg = builder.build(); + TestRecordsTextProto.SimpleDocument.Builder newBuilder = TestRecordsTextProto.SimpleDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "text", "suggestion"); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(1)); + TestRecordsTextProto.SimpleDocument partialMsg = newBuilder.build(); // The suggestion is supposed to show up in text field assertEquals("suggestion", partialMsg.getText()); @@ -79,24 +87,28 @@ void simple() { @Test void group() { - TestRecordsTextProto.SimpleDocument message = TestRecordsTextProto.SimpleDocument.newBuilder() + TestRecordsTextProto.SimpleDocument.Builder builder = TestRecordsTextProto.SimpleDocument.newBuilder() .setDocId(2) .setText("more text") - .setGroup(2) - .build(); + .setGroup(2); + TestRecordsTextProto.SimpleDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = function(LuceneFunctionNames.LUCENE_TEXT, field("text")).groupBy(field("group")); assertEquals(ImmutableMap.of(Tuple.from(2), ImmutableList.of(textField("text", "more text"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "text" for text field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("text", Set.of("text")), analyzerProvider); + assertEquals("more text", builder.build().getText()); + KeyExpression primaryKey = concat(field("group"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.SimpleDocument.Builder builder = TestRecordsTextProto.SimpleDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "text", "suggestion", Tuple.from(2)); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(2, 2)); - TestRecordsTextProto.SimpleDocument partialMsg = builder.build(); + TestRecordsTextProto.SimpleDocument.Builder newBuilder = TestRecordsTextProto.SimpleDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "text", "suggestion", Tuple.from(2)); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(2, 2)); + TestRecordsTextProto.SimpleDocument partialMsg = newBuilder.build(); // The suggestion is supposed to show up in text field assertEquals("suggestion", partialMsg.getText()); @@ -110,11 +122,11 @@ void group() { @Test void multi() { - TestRecordsTextProto.MultiDocument message = TestRecordsTextProto.MultiDocument.newBuilder() + TestRecordsTextProto.MultiDocument.Builder builder = TestRecordsTextProto.MultiDocument.newBuilder() .setDocId(3) .addText("some text") - .addText("other text") - .build(); + .addText("other text"); + TestRecordsTextProto.MultiDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = function(LuceneFunctionNames.LUCENE_TEXT, field("text", KeyExpression.FanType.FanOut)); assertEquals(ImmutableMap.of(Tuple.from(), ImmutableList.of( @@ -122,14 +134,19 @@ void multi() { textField("text", "other text"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "text" for text field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("text", Set.of("text")), analyzerProvider); + assertEquals("some text", builder.build().getText(0)); + assertEquals("other text", builder.build().getText(1)); + KeyExpression primaryKey = field("doc_id"); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.MultiDocument.Builder builder = TestRecordsTextProto.MultiDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "text", "suggestion", Tuple.from(2)); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(3)); - TestRecordsTextProto.MultiDocument partialMsg = builder.build(); + TestRecordsTextProto.MultiDocument.Builder newBuilder = TestRecordsTextProto.MultiDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "text", "suggestion", Tuple.from(2)); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(3)); + TestRecordsTextProto.MultiDocument partialMsg = newBuilder.build(); // The suggestion is supposed to show up in text field assertEquals(1, partialMsg.getTextCount()); @@ -141,7 +158,7 @@ void multi() { @Test void biGroup() { - TestRecordsTextProto.ComplexDocument message = TestRecordsTextProto.ComplexDocument.newBuilder() + TestRecordsTextProto.ComplexDocument.Builder builder = TestRecordsTextProto.ComplexDocument.newBuilder() .setHeader(TestRecordsTextProto.ComplexDocument.Header.newBuilder().setHeaderId(4)) .setGroup(10) .setDocId(4) @@ -149,8 +166,8 @@ void biGroup() { .addTag("tag1") .addTag("tag2") .setText2("second text") - .setScore(100) - .build(); + .setScore(100); + TestRecordsTextProto.ComplexDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = concat( function(LuceneFunctionNames.LUCENE_TEXT, field("text")), @@ -162,14 +179,19 @@ void biGroup() { Tuple.from(10, "tag2"), ImmutableList.of(textField("text", "first text"), textField("text2", "second text"), intField("score", 100))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "text" for text field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("text2", Set.of("text")), analyzerProvider); + assertEquals("first text", builder.build().getText()); + assertEquals("second text", builder.build().getText2()); + KeyExpression primaryKey = concat(field("group"), field("header").nest("header_id"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.ComplexDocument.Builder builder = TestRecordsTextProto.ComplexDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "text", "suggestion", Tuple.from(10, "tag1")); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(10, 4, 4)); - TestRecordsTextProto.ComplexDocument partialMsg = builder.build(); + TestRecordsTextProto.ComplexDocument.Builder newBuilder = TestRecordsTextProto.ComplexDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "text", "suggestion", Tuple.from(10, "tag1")); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(10, 4, 4)); + TestRecordsTextProto.ComplexDocument partialMsg = newBuilder.build(); // The suggestion is supposed to show up in text field assertEquals("suggestion", partialMsg.getText()); @@ -187,12 +209,12 @@ void biGroup() { @Test void uncorrelatedMap() { - TestRecordsTextProto.MapDocument message = TestRecordsTextProto.MapDocument.newBuilder() + TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder() .setGroup(10) .setDocId(5) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k1").setValue("v1")) - .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k2").setValue("v2")) - .build(); + .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k2").setValue("v2")); + TestRecordsTextProto.MapDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = field("entry", KeyExpression.FanType.FanOut).nest(concat(field("key"), function(LuceneFunctionNames.LUCENE_TEXT, field("value")))); assertEquals(ImmutableMap.of(Tuple.from(), ImmutableList.of( @@ -202,14 +224,19 @@ void uncorrelatedMap() { textField("entry_value", "v2"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "v2" for entry_value field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("entry_value", Set.of("v2")), analyzerProvider); + assertEquals("v1", builder.build().getEntry(0).getValue()); + assertEquals("v2", builder.build().getEntry(1).getValue()); + KeyExpression primaryKey = concat(field("group"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "entry_value", "suggestion"); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(10, 5)); - TestRecordsTextProto.MapDocument partialMsg = builder.build(); + TestRecordsTextProto.MapDocument.Builder newBuilder = TestRecordsTextProto.MapDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "entry_value", "suggestion"); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(10, 5)); + TestRecordsTextProto.MapDocument partialMsg = newBuilder.build(); assertEquals(1, partialMsg.getEntryCount()); TestRecordsTextProto.MapDocument.Entry entry = partialMsg.getEntry(0); @@ -223,12 +250,12 @@ void uncorrelatedMap() { @Test void map() { - TestRecordsTextProto.MapDocument message = TestRecordsTextProto.MapDocument.newBuilder() + TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder() .setGroup(10) .setDocId(5) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k1").setValue("v1")) - .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k2").setValue("v2")) - .build(); + .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k2").setValue("v2")); + TestRecordsTextProto.MapDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = function(LuceneFunctionNames.LUCENE_FIELD_NAME, concat( field("entry", KeyExpression.FanType.FanOut).nest(function(LuceneFunctionNames.LUCENE_FIELD_NAME, concat(function(LuceneFunctionNames.LUCENE_TEXT, field("value")), field("key")))), @@ -238,14 +265,19 @@ void map() { textField("k2", "v2"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "v2" for k2 field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("k2", Set.of("v2")), analyzerProvider); + assertEquals("v1", builder.build().getEntry(0).getValue()); + assertEquals("v2", builder.build().getEntry(1).getValue()); + KeyExpression primaryKey = concat(field("group"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "k1", "suggestion"); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(10, 5)); - TestRecordsTextProto.MapDocument partialMsg = builder.build(); + TestRecordsTextProto.MapDocument.Builder newBuilder = TestRecordsTextProto.MapDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "k1", "suggestion"); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(10, 5)); + TestRecordsTextProto.MapDocument partialMsg = newBuilder.build(); assertEquals(1, partialMsg.getEntryCount()); TestRecordsTextProto.MapDocument.Entry entry = partialMsg.getEntry(0); @@ -261,12 +293,12 @@ void map() { @Test void groupedMap() { - TestRecordsTextProto.MapDocument message = TestRecordsTextProto.MapDocument.newBuilder() + TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder() .setDocId(6) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k1").setValue("v10")) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("k2").setValue("v20")) - .setGroup(20) - .build(); + .setGroup(20); + TestRecordsTextProto.MapDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = function(LuceneFunctionNames.LUCENE_FIELD_NAME, concat( field("entry", KeyExpression.FanType.FanOut).nest(function(LuceneFunctionNames.LUCENE_FIELD_NAME, concat(function(LuceneFunctionNames.LUCENE_TEXT, field("value")), field("key")))), @@ -277,14 +309,19 @@ void groupedMap() { textField("k2", "v20"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "v20" for k2 field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("k2", Set.of("v20")), analyzerProvider); + assertEquals("v10", builder.build().getEntry(0).getValue()); + assertEquals("v20", builder.build().getEntry(1).getValue()); + KeyExpression primaryKey = concat(field("group"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "k1", "suggestion", Tuple.from(20)); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(20, 6)); - TestRecordsTextProto.MapDocument partialMsg = builder.build(); + TestRecordsTextProto.MapDocument.Builder newBuilder = TestRecordsTextProto.MapDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "k1", "suggestion", Tuple.from(20)); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(20, 6)); + TestRecordsTextProto.MapDocument partialMsg = newBuilder.build(); assertEquals(1, partialMsg.getEntryCount()); TestRecordsTextProto.MapDocument.Entry entry = partialMsg.getEntry(0); @@ -303,12 +340,12 @@ void groupedMap() { @Test void groupingMap() { - TestRecordsTextProto.MapDocument message = TestRecordsTextProto.MapDocument.newBuilder() + TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder() .setDocId(7) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("r1").setValue("val").setSecondValue("2val").setThirdValue("3val")) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("r2").setValue("nval").setSecondValue("2nval").setThirdValue("3nval")) - .setGroup(30) - .build(); + .setGroup(30); + TestRecordsTextProto.MapDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = new GroupingKeyExpression(concat(field("group"), field("entry", KeyExpression.FanType.FanOut) @@ -321,14 +358,21 @@ void groupingMap() { Tuple.from(30, "r2"), ImmutableList.of(textField("entry_value", "nval"), textField("entry_second_value", "2nval"), textField("entry_third_value", "3nval"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "2val" for entry_second_value field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("entry_second_value", Set.of("2val")), analyzerProvider); + assertEquals("val", builder.build().getEntry(0).getValue()); + assertEquals("2val", builder.build().getEntry(0).getSecondValue()); + assertEquals("nval", builder.build().getEntry(1).getValue()); + assertEquals("2nval", builder.build().getEntry(1).getSecondValue()); + KeyExpression primaryKey = concat(field("group"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "entry_value", "suggestion", Tuple.from(30, "r1")); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(30, 7)); - TestRecordsTextProto.MapDocument partialMsg = builder.build(); + TestRecordsTextProto.MapDocument.Builder newBuilder = TestRecordsTextProto.MapDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "entry_value", "suggestion", Tuple.from(30, "r1")); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(30, 7)); + TestRecordsTextProto.MapDocument partialMsg = newBuilder.build(); assertEquals(1, partialMsg.getEntryCount()); TestRecordsTextProto.MapDocument.Entry entry = partialMsg.getEntry(0); @@ -349,13 +393,13 @@ void groupingMap() { @Test void groupingMapWithExtra() { - TestRecordsTextProto.MapDocument message = TestRecordsTextProto.MapDocument.newBuilder() + TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder() .setDocId(8) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("en").setValue("first").setSecondValue("second")) .addEntry(TestRecordsTextProto.MapDocument.Entry.newBuilder().setKey("de").setValue("erste").setSecondValue("zweite")) .setGroup(40) - .setText2("extra") - .build(); + .setText2("extra"); + TestRecordsTextProto.MapDocument message = builder.build(); FDBRecord record = unstoredRecord(message); KeyExpression index = new GroupingKeyExpression(concat( field("group"), @@ -366,14 +410,21 @@ void groupingMapWithExtra() { Tuple.from(40, "de"), ImmutableList.of(textField("entry_value", "erste"), textField("entry_second_value", "zweite"), textField("text2", "extra"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "second" for entry_second_value field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("entry_second_value", Set.of("second")), analyzerProvider); + assertEquals("first", builder.build().getEntry(0).getValue()); + assertEquals("second", builder.build().getEntry(0).getSecondValue()); + assertEquals("erste", builder.build().getEntry(1).getValue()); + assertEquals("zweite", builder.build().getEntry(1).getSecondValue()); + KeyExpression primaryKey = concat(field("group"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.MapDocument.Builder builder = TestRecordsTextProto.MapDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "entry_second_value", "suggestion", Tuple.from(40, "en")); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(40, 8)); - TestRecordsTextProto.MapDocument partialMsg = builder.build(); + TestRecordsTextProto.MapDocument.Builder newBuilder = TestRecordsTextProto.MapDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "entry_second_value", "suggestion", Tuple.from(40, "en")); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(40, 8)); + TestRecordsTextProto.MapDocument partialMsg = newBuilder.build(); assertEquals(1, partialMsg.getEntryCount()); TestRecordsTextProto.MapDocument.Entry entry = partialMsg.getEntry(0); @@ -394,11 +445,11 @@ void groupingMapWithExtra() { @Test void mapWithSubMessage() { - TestRecordsTextProto.NestedMapDocument message = TestRecordsTextProto.NestedMapDocument.newBuilder() + TestRecordsTextProto.NestedMapDocument.Builder builder = TestRecordsTextProto.NestedMapDocument.newBuilder() .setDocId(5) .setGroup(50) - .addEntry(TestRecordsTextProto.NestedMapDocument.Entry.newBuilder().setKey("k1").setSubEntry(TestRecordsTextProto.NestedMapDocument.SubEntry.newBuilder().setValue("testValue").build()).build()) - .build(); + .addEntry(TestRecordsTextProto.NestedMapDocument.Entry.newBuilder().setKey("k1").setSubEntry(TestRecordsTextProto.NestedMapDocument.SubEntry.newBuilder().setValue("testValue").build()).build()); + TestRecordsTextProto.NestedMapDocument message = builder.build(); KeyExpression index = field("entry", KeyExpression.FanType.FanOut) .nest(function(LuceneFunctionNames.LUCENE_FIELD_NAME, concat(field("sub_entry").nest(concat(function(LuceneFunctionNames.LUCENE_TEXT, field("value")), function(LuceneFunctionNames.LUCENE_TEXT, field("second_value")))), @@ -409,14 +460,18 @@ void mapWithSubMessage() { textField("entry_k1_value", "testValue"))), LuceneDocumentFromRecord.getRecordFields(index, record)); + // Highlight "testValue" for entry_k1_value field + LuceneDocumentFromRecord.highlightTermsInMessage(index, builder, Map.of("entry_k1_value", Set.of("testvalue")), analyzerProvider); + assertEquals("testValue", builder.build().getEntry(0).getSubEntry().getValue()); + KeyExpression primaryKey = concat(field("group"), field("doc_id")); // Build the partial record message for suggestion Descriptors.Descriptor recordDescriptor = message.getDescriptorForType(); - TestRecordsTextProto.NestedMapDocument.Builder builder = TestRecordsTextProto.NestedMapDocument.newBuilder(); - LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, builder, "entry_k1_value", "suggestion"); - LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, builder, Tuple.from(50, 5)); - TestRecordsTextProto.NestedMapDocument partialMsg = builder.build(); + TestRecordsTextProto.NestedMapDocument.Builder newBuilder = TestRecordsTextProto.NestedMapDocument.newBuilder(); + LuceneIndexKeyValueToPartialRecordUtils.buildPartialRecord(index, recordDescriptor, newBuilder, "entry_k1_value", "suggestion"); + LuceneIndexKeyValueToPartialRecordUtils.populatePrimaryKey(primaryKey, recordDescriptor, newBuilder, Tuple.from(50, 5)); + TestRecordsTextProto.NestedMapDocument partialMsg = newBuilder.build(); assertEquals(1, partialMsg.getEntryCount()); TestRecordsTextProto.NestedMapDocument.Entry entry = partialMsg.getEntry(0); diff --git a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java index 5ff5fdcd79..8c919b8e93 100644 --- a/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java +++ b/fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTest.java @@ -340,9 +340,14 @@ protected FDBRecordContextConfig.Builder contextConfig(@Nonnull final RecordLaye } private LuceneScanBounds fullTextSearch(Index index, String search) { + return fullTextSearch(index, search, false); + } + + private LuceneScanBounds fullTextSearch(Index index, String search, boolean highlight) { LuceneScanParameters scan = new LuceneScanQueryParameters( ScanComparisons.EMPTY, - new LuceneQueryMultiFieldSearchClause(search, false)); + new LuceneQueryMultiFieldSearchClause(search, false), + null, null, null, highlight); return scan.bind(recordStore, index, EvaluationContext.EMPTY); }