Skip to content

Commit

Permalink
Annotated text fields are stored by default with synthetic source (#1…
Browse files Browse the repository at this point in the history
…07922)

This change follows existing implementation for text field.

Closes #107734.
  • Loading branch information
lkts committed Apr 26, 2024
1 parent 01cc967 commit ca513b1
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 10 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/107922.yaml
@@ -0,0 +1,6 @@
pr: 107922
summary: Feature/annotated text store defaults
area: Mapping
type: enhancement
issues:
- 107734
Expand Up @@ -83,20 +83,24 @@ private static NamedAnalyzer wrapAnalyzer(NamedAnalyzer in) {

public static class Builder extends FieldMapper.Builder {

private final Parameter<Boolean> store = Parameter.storeParam(m -> builder(m).store.getValue(), false);

final TextParams.Analyzers analyzers;
final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> builder(m).similarity.getValue());

final Parameter<String> indexOptions = TextParams.textIndexOptions(m -> builder(m).indexOptions.getValue());
final Parameter<Boolean> norms = TextParams.norms(true, m -> builder(m).norms.getValue());
final Parameter<String> termVectors = TextParams.termVectors(m -> builder(m).termVectors.getValue());

private final Parameter<Map<String, String>> meta = Parameter.metaParam();

private final IndexVersion indexCreatedVersion;
private final TextParams.Analyzers analyzers;
private final boolean isSyntheticSourceEnabledViaIndexMode;
private final Parameter<Boolean> store;

public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
public Builder(
String name,
IndexVersion indexCreatedVersion,
IndexAnalyzers indexAnalyzers,
boolean isSyntheticSourceEnabledViaIndexMode
) {
super(name);
this.indexCreatedVersion = indexCreatedVersion;
this.analyzers = new TextParams.Analyzers(
Expand All @@ -105,6 +109,11 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
m -> builder(m).analyzers.positionIncrementGap.getValue(),
indexCreatedVersion
);
this.isSyntheticSourceEnabledViaIndexMode = isSyntheticSourceEnabledViaIndexMode;
this.store = Parameter.storeParam(
m -> builder(m).store.getValue(),
() -> isSyntheticSourceEnabledViaIndexMode && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false
);
}

@Override
Expand Down Expand Up @@ -164,7 +173,9 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) {
}
}

public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
public static TypeParser PARSER = new TypeParser(
(n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.getIndexSettings().getMode().isSyntheticSourceEnabled())
);

/**
* Parses markdown-like syntax into plain text and AnnotationTokens with offsets for
Expand Down Expand Up @@ -552,7 +563,12 @@ protected String contentType() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
return new Builder(
simpleName(),
builder.indexCreatedVersion,
builder.analyzers.indexAnalyzers,
builder.isSyntheticSourceEnabledViaIndexMode
).init(this);
}

@Override
Expand Down
Expand Up @@ -24,7 +24,9 @@
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.CharFilterFactory;
Expand All @@ -42,6 +44,7 @@
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.TextFieldFamilySyntheticSourceTestSetup;
import org.elasticsearch.index.mapper.TextFieldMapper;
import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.xcontent.ToXContent;
import org.elasticsearch.xcontent.XContentBuilder;
Expand Down Expand Up @@ -288,6 +291,64 @@ public void testEnableStore() throws IOException {
assertTrue(fields.get(0).fieldType().stored());
}

public void testStoreParameterDefaults() throws IOException {
var timeSeriesIndexMode = randomBoolean();
var isStored = randomBoolean();
var hasKeywordFieldForSyntheticSource = randomBoolean();

var indexSettingsBuilder = getIndexSettingsBuilder();
if (timeSeriesIndexMode) {
indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
.putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-08T23:40:53.384Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z");
}
var indexSettings = indexSettingsBuilder.build();

var mapping = mapping(b -> {
b.startObject("field");
b.field("type", "annotated_text");
if (isStored) {
b.field("store", isStored);
}
if (hasKeywordFieldForSyntheticSource) {
b.startObject("fields");
b.startObject("keyword");
b.field("type", "keyword");
b.endObject();
b.endObject();
}
b.endObject();

if (timeSeriesIndexMode) {
b.startObject("@timestamp");
b.field("type", "date");
b.endObject();
b.startObject("dimension");
b.field("type", "keyword");
b.field("time_series_dimension", "true");
b.endObject();
}
});
DocumentMapper mapper = createMapperService(getVersion(), indexSettings, () -> true, mapping).documentMapper();

var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
b.field("field", "1234");
if (timeSeriesIndexMode) {
b.field("@timestamp", randomMillisUpToYear9999());
b.field("dimension", "dimension1");
}
}, null);
ParsedDocument doc = mapper.parse(source);
List<IndexableField> fields = doc.rootDoc().getFields("field");
IndexableFieldType fieldType = fields.get(0).fieldType();
if (isStored || (timeSeriesIndexMode && hasKeywordFieldForSyntheticSource == false)) {
assertTrue(fieldType.stored());
} else {
assertFalse(fieldType.stored());
}
}

public void testDisableNorms() throws IOException {

DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> {
Expand Down
Expand Up @@ -29,9 +29,12 @@ public void testIntervals() throws IOException {
}

public void testFetchSourceValue() throws IOException {
MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", IndexVersion.current(), createDefaultIndexAnalyzers())
.build(MapperBuilderContext.root(false, false))
.fieldType();
MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder(
"field",
IndexVersion.current(),
createDefaultIndexAnalyzers(),
false
).build(MapperBuilderContext.root(false, false)).fieldType();

assertEquals(List.of("value"), fetchSourceValue(fieldType, "value"));
assertEquals(List.of("42"), fetchSourceValue(fieldType, 42L));
Expand Down

0 comments on commit ca513b1

Please sign in to comment.