Skip to content

Commit

Permalink
revert auto-complete field combining at index time
Browse files Browse the repository at this point in the history
  • Loading branch information
@alecgrieser
alecgrieser committed May 19, 2022
1 parent 7a7d8bd commit 302db72
Show file tree
Hide file tree
Showing 5 changed files with 18 additions and 53 deletions.
13 changes: 3 additions & 10 deletions ...r-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ public void addField(@Nonnull T source, @Nonnull String fieldName, @Nullable fin
LuceneIndexExpressions.DocumentFieldType type,
boolean stored, boolean sorted,
@Nonnull List<Integer> overriddenKeyRanges, int groupingKeyIndex,
@Nonnull Map<String, Object> fieldConfigs, boolean nested) {
fields.add(new DocumentField(fieldName, value, type, stored, sorted, fieldConfigs, nested));
@Nonnull Map<String, Object> fieldConfigs) {
fields.add(new DocumentField(fieldName, value, type, stored, sorted, fieldConfigs));
}
}

Expand All @@ -197,19 +197,17 @@ protected static class DocumentField {
private final LuceneIndexExpressions.DocumentFieldType type;
private final boolean stored;
private final boolean sorted;
private final boolean nested;
@Nonnull
private final Map<String, Object> fieldConfigs;

public DocumentField(@Nonnull String fieldName, @Nullable Object value, LuceneIndexExpressions.DocumentFieldType type,
boolean stored, boolean sorted, @Nonnull Map<String, Object> fieldConfigs, boolean nested) {
boolean stored, boolean sorted, @Nonnull Map<String, Object> fieldConfigs) {
this.fieldName = fieldName;
this.value = value;
this.type = type;
this.stored = stored;
this.sorted = sorted;
this.fieldConfigs = fieldConfigs;
this.nested = nested;
}

@Nonnull
Expand All @@ -234,10 +232,6 @@ public boolean isSorted() {
return sorted;
}

public boolean isNested() {
return nested;
}

@Nullable
public Object getConfig(@Nonnull String key) {
return fieldConfigs.get(key);
Expand Down Expand Up @@ -278,7 +272,6 @@ public int hashCode() {
result = 31 * result + type.hashCode();
result = 31 * result + (stored ? 1 : 0);
result = 31 * result + (sorted ? 1 : 0);
result = 31 * result + (nested ? 1 : 0);
return result;
}

Expand Down
17 changes: 8 additions & 9 deletions ...yer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexExpressions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public enum DocumentFieldType { STRING, TEXT, INT, LONG, DOUBLE, BOOLEAN }
* @param recordType Protobuf meta-data for record type
*/
public static void validate(@Nonnull KeyExpression root, @Nonnull Descriptors.Descriptor recordType) {
getFields(root, new MetaDataSource(recordType), (source, fieldName, value, type, stored, sorted, overriddeKeyRanges, groupingKeyIndex, fieldConfigsIgnored, nested) -> {
getFields(root, new MetaDataSource(recordType), (source, fieldName, value, type, stored, sorted, overriddeKeyRanges, groupingKeyIndex, fieldConfigsIgnored) -> {
}, null);
}

Expand Down Expand Up @@ -152,7 +152,7 @@ public static Map<String, DocumentFieldDerivation> getDocumentFieldDerivations(@
final Map<String, DocumentFieldDerivation> fields = new HashMap<>();
getFields(root,
new MetaDataSource(recordType),
(source, fieldName, value, type, stored, sorted, overriddenKeyRanges, groupingKeyIndex, fieldConfigsIgnored, nested) -> {
(source, fieldName, value, type, stored, sorted, overriddenKeyRanges, groupingKeyIndex, fieldConfigsIgnored) -> {
List<String> path = new ArrayList<>();
for (MetaDataSource metaDataSource = source; metaDataSource != null; metaDataSource = metaDataSource.getParent()) {
if (metaDataSource.getField() != null) {
Expand Down Expand Up @@ -188,8 +188,7 @@ public interface RecordSource<T extends RecordSource<T>> {
public interface DocumentDestination<T extends RecordSource<T>> {
@SuppressWarnings("java:S107")
void addField(@Nonnull T source, @Nonnull String fieldName, @Nullable Object value, @Nonnull DocumentFieldType type,
boolean stored, boolean sorted, @Nonnull List<Integer> overriddenKeyRanges, int groupingKeyIndex,
@Nonnull Map<String, Object> fieldConfigs, boolean nested);
boolean stored, boolean sorted, @Nonnull List<Integer> overriddenKeyRanges, int groupingKeyIndex, @Nonnull Map<String, Object> fieldConfigs);
}

/**
Expand All @@ -208,18 +207,18 @@ public static <T extends RecordSource<T>> void getFields(@Nonnull KeyExpression
expression = root;
}
getFieldsRecursively(expression, source, destination, fieldNamePrefix, 0,
root instanceof GroupingKeyExpression ? ((GroupingKeyExpression) root).getGroupingCount() : 0, new ArrayList<>(), false);
root instanceof GroupingKeyExpression ? ((GroupingKeyExpression) root).getGroupingCount() : 0, new ArrayList<>());
}

@SuppressWarnings("squid:S3776")
public static <T extends RecordSource<T>> void getFieldsRecursively(@Nonnull KeyExpression expression,
@Nonnull T source, @Nonnull DocumentDestination<T> destination,
@Nullable String fieldNamePrefix, int keyIndex, int groupingCount,
@Nonnull List<Integer> overriddenKeyRanges, boolean nested) {
@Nonnull List<Integer> overriddenKeyRanges) {
if (expression instanceof ThenKeyExpression) {
int count = 0;
for (KeyExpression child : ((ThenKeyExpression)expression).getChildren()) {
getFieldsRecursively(child, source, destination, fieldNamePrefix, keyIndex + count, groupingCount, overriddenKeyRanges, nested);
getFieldsRecursively(child, source, destination, fieldNamePrefix, keyIndex + count, groupingCount, overriddenKeyRanges);
count += child.getColumnSize();
}
return;
Expand Down Expand Up @@ -258,7 +257,7 @@ public static <T extends RecordSource<T>> void getFieldsRecursively(@Nonnull Key
}
String fieldName = appendFieldName(fieldNamePrefix, fieldNameSuffix);
for (T subsource : source.getChildren(parentExpression)) {
getFieldsRecursively(child, subsource, destination, fieldName, keyIndex, groupingCount, overriddenKeyRanges, true);
getFieldsRecursively(child, subsource, destination, fieldName, keyIndex, groupingCount, overriddenKeyRanges);
}
if (suffixOverride) {
// Remove the last 2 numbers added above
Expand Down Expand Up @@ -336,7 +335,7 @@ public static <T extends RecordSource<T>> void getFieldsRecursively(@Nonnull Key
}
for (Object value : source.getValues(fieldExpression)) {
destination.addField(source, fieldName, value, fieldType, fieldStored, fieldSorted,
overriddenKeyRanges, keyIndex < groupingCount ? keyIndex : -1, configs, nested);
overriddenKeyRanges, keyIndex < groupingCount ? keyIndex : -1, configs);
}
if (suffixOverride) {
// Remove the last 2 numbers added above
Expand Down
4 changes: 2 additions & 2 deletions ...in/java/com/apple/foundationdb/record/lucene/LuceneIndexKeyValueToPartialRecordUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public static void buildPartialRecord(@Nonnull KeyExpression root, @Nonnull Desc
@Nonnull String suggestion, @Nonnull Tuple groupingKey) {
final KeyExpression expression = root instanceof GroupingKeyExpression ? ((GroupingKeyExpression) root).getWholeKey() : root;
LuceneIndexExpressions.getFieldsRecursively(expression, new PartialRecordBuildSource(null, descriptor, builder),
(source, fieldName, value, type, stored, sorted, overriddenKeyRanges, groupingKeyIndex, fieldConfigsIgnored, nested) -> {
(source, fieldName, value, type, stored, sorted, overriddenKeyRanges, groupingKeyIndex, fieldConfigsIgnored) -> {
if (groupingKeyIndex > - 1) {
if (groupingKeyIndex > groupingKey.size() - 1) {
throw new RecordCoreException("Invalid grouping value tuple given a grouping key")
Expand All @@ -68,7 +68,7 @@ public static void buildPartialRecord(@Nonnull KeyExpression root, @Nonnull Desc
buildIfFieldNameMatch(source, fieldName, luceneField, overriddenKeyRanges, suggestion, (String) value);
}
},
null, 0, root instanceof GroupingKeyExpression ? ((GroupingKeyExpression) root).getGroupingCount() : 0, new ArrayList<>(), false);
null, 0, root instanceof GroupingKeyExpression ? ((GroupingKeyExpression) root).getGroupingCount() : 0, new ArrayList<>());
}

private static void buildIfFieldNameMatch(@Nonnull PartialRecordBuildSource source, @Nonnull String concatenatedFieldPath, @Nonnull String givenFieldName,
Expand Down
27 changes: 0 additions & 27 deletions ...ayer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
import com.apple.foundationdb.record.provider.foundationdb.indexes.StandardIndexMaintainer;
import com.apple.foundationdb.record.query.QueryToKeyMatcher;
import com.apple.foundationdb.tuple.Tuple;
import com.google.common.base.Joiner;
import com.google.protobuf.Message;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
Expand Down Expand Up @@ -222,15 +221,6 @@ private void writeDocument(@Nonnull List<LuceneDocumentFromRecord.DocumentField>
.filter(f -> f.getType().equals(LuceneIndexExpressions.DocumentFieldType.TEXT))
.map(f -> (String) f.getValue()).collect(Collectors.toList());
Document document = new Document();
if (autoCompleteEnabled) {
List<String> nestedFields = fields.stream()
.filter(f -> f.isNested())
.filter(f -> f.getType().equals(LuceneIndexExpressions.DocumentFieldType.TEXT))
.map(f -> (String) f.getValue()).collect(Collectors.toList());
if (nestedFields.size() > 0) {
document.add(new Field("TEXT_NESTED", Joiner.on(" ").join(nestedFields), getAutoCompleteTextFieldType()));
}
}
final IndexWriter newWriter = directoryManager.getIndexWriter(groupingKey,
indexAnalyzerChooser.chooseAnalyzer(texts));
BytesRef ref = new BytesRef(primaryKey);
Expand Down Expand Up @@ -338,23 +328,6 @@ private FieldType getTextFieldType(LuceneDocumentFromRecord.DocumentField field)
return ft;
}

private FieldType getAutoCompleteTextFieldType() {
FieldType ft = new FieldType();

try {
ft.setIndexOptions(IndexOptions.DOCS);
ft.setTokenized(true);
ft.setStored(false);
ft.setStoreTermVectors(false);
ft.setOmitNorms(true);
ft.freeze();
} catch (ClassCastException ex) {
throw new RecordCoreArgumentException("Invalid value type for Lucene field config", ex);
}

return ft;
}

private static IndexOptions getIndexOptions(@Nonnull String value) {
try {
return IndexOptions.valueOf(value);
Expand Down
10 changes: 5 additions & 5 deletions ...cene/src/test/java/com/apple/foundationdb/record/lucene/LuceneDocumentFromRecordTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,25 +473,25 @@ private static FDBRecord<Message> unstoredRecord(Message message) {

private static LuceneDocumentFromRecord.DocumentField documentField(String name, @Nullable Object value, LuceneIndexExpressions.DocumentFieldType type,
boolean stored, boolean sorted,
@Nonnull Map<String, Object> fieldConfigs, boolean nested) {
return new LuceneDocumentFromRecord.DocumentField(name, value, type, stored, sorted, fieldConfigs, nested);
@Nonnull Map<String, Object> fieldConfigs) {
return new LuceneDocumentFromRecord.DocumentField(name, value, type, stored, sorted, fieldConfigs);
}

private static LuceneDocumentFromRecord.DocumentField stringField(String name, String value) {
return documentField(name, value, LuceneIndexExpressions.DocumentFieldType.STRING, false, false, Collections.emptyMap(), false);
return documentField(name, value, LuceneIndexExpressions.DocumentFieldType.STRING, false, false, Collections.emptyMap());
}

private static LuceneDocumentFromRecord.DocumentField textField(String name, String value) {
return textField(name, value, Collections.emptyMap());
}

private static LuceneDocumentFromRecord.DocumentField textField(String name, String value, Map<String, Object> fieldConfigs) {
return documentField(name, value, LuceneIndexExpressions.DocumentFieldType.TEXT, false, false, fieldConfigs, false);
return documentField(name, value, LuceneIndexExpressions.DocumentFieldType.TEXT, false, false, fieldConfigs);
}


private static LuceneDocumentFromRecord.DocumentField intField(String name, int value) {
return documentField(name, value, LuceneIndexExpressions.DocumentFieldType.INT, false, false, Collections.emptyMap(), false);
return documentField(name, value, LuceneIndexExpressions.DocumentFieldType.INT, false, false, Collections.emptyMap());
}

}

0 comments on commit 302db72

Please sign in to comment.