Add runtime fields/mappings

.github/workflows/codeql-v7.yml

@@ -14,7 +14,7 @@ jobs:
   codeql:
     strategy:
       matrix:
-        go: [1.15.x, 1.16.x]
+        go: [1.16.x, 1.17.x]
         os: [ubuntu-latest]
     name: Run ${{ matrix.go }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}

.github/workflows/test-v7.yml

@@ -10,7 +10,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go: [1.15.x, 1.16.x]
+        go: [1.16.x, 1.17.x]
         os: [ubuntu-latest]
     name: Run ${{ matrix.go }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}

docker-compose.cluster.yml

@@ -1,6 +1,6 @@
 services:
   es1:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
     hostname: es1
     environment:
       - bootstrap.memory_lock=true
@@ -26,7 +26,7 @@ services:
       - 9200:9200
 
   es2:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
     hostname: es2
     environment:
       - bootstrap.memory_lock=true
@@ -52,7 +52,7 @@ services:
       - 9201:9200
 
   es3:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
     hostname: es3
     environment:
       - bootstrap.memory_lock=true

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
     hostname: elasticsearch
     environment:
       - cluster.name=elasticsearch
@@ -28,7 +28,7 @@ services:
     ports:
       - 9200:9200
   platinum:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.13.4}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${VERSION:-7.14.0}
     hostname: elasticsearch-platinum
     environment:
       - cluster.name=platinum

field_caps.go

@@ -32,6 +32,7 @@ type FieldCapsService struct {
 	expandWildcards   string
 	fields            []string
 	ignoreUnavailable *bool
+	includeUnmapped   *bool
 	bodyJson          interface{}
 	bodyString        string
 }
@@ -117,6 +118,12 @@ func (s *FieldCapsService) IgnoreUnavailable(ignoreUnavailable bool) *FieldCapsS
 	return s
 }
 
+// IncludeUnmapped specifies whether unmapped fields whould be included in the response.
+func (s *FieldCapsService) IncludeUnmapped(includeUnmapped bool) *FieldCapsService {
+	s.includeUnmapped = &includeUnmapped
+	return s
+}
+
 // BodyJson is documented as: Field json objects containing the name and optionally a range to filter out indices result, that have results outside the defined bounds.
 func (s *FieldCapsService) BodyJson(body interface{}) *FieldCapsService {
 	s.bodyJson = body
@@ -160,7 +167,7 @@ func (s *FieldCapsService) buildURL() (string, url.Values, error) {
 		params.Set("filter_path", strings.Join(s.filterPath, ","))
 	}
 	if s.allowNoIndices != nil {
-		params.Set("allow_no_indices", fmt.Sprintf("%v", *s.allowNoIndices))
+		params.Set("allow_no_indices", fmt.Sprint(*s.allowNoIndices))
 	}
 	if s.expandWildcards != "" {
 		params.Set("expand_wildcards", s.expandWildcards)
@@ -169,7 +176,10 @@ func (s *FieldCapsService) buildURL() (string, url.Values, error) {
 		params.Set("fields", strings.Join(s.fields, ","))
 	}
 	if s.ignoreUnavailable != nil {
-		params.Set("ignore_unavailable", fmt.Sprintf("%v", *s.ignoreUnavailable))
+		params.Set("ignore_unavailable", fmt.Sprint(*s.ignoreUnavailable))
+	}
+	if s.includeUnmapped != nil {
+		params.Set("include_unmapped", fmt.Sprint(*s.includeUnmapped))
 	}
 	return path, params, nil
 }
@@ -231,7 +241,9 @@ func (s *FieldCapsService) Do(ctx context.Context) (*FieldCapsResponse, error) {
 // FieldCapsRequest can be used to set up the body to be used in the
 // Field Capabilities API.
 type FieldCapsRequest struct {
-	Fields []string `json:"fields"`
+	Fields          []string        `json:"fields"` // list of fields to retrieve
+	IndexFilter     Query           `json:"index_filter,omitempty"`
+	RuntimeMappings RuntimeMappings `json:"runtime_mappings,omitempty"`
 }
 
 // -- Response --
@@ -248,10 +260,12 @@ type FieldCapsType map[string]FieldCaps // type -> caps
 
 // FieldCaps contains capabilities of an individual field.
 type FieldCaps struct {
-	Type                   string   `json:"type"`
-	Searchable             bool     `json:"searchable"`
-	Aggregatable           bool     `json:"aggregatable"`
-	Indices                []string `json:"indices,omitempty"`
-	NonSearchableIndices   []string `json:"non_searchable_indices,omitempty"`
-	NonAggregatableIndices []string `json:"non_aggregatable_indices,omitempty"`
+	Type                   string                 `json:"type"`
+	MetadataField          bool                   `json:"metadata_field"`
+	Searchable             bool                   `json:"searchable"`
+	Aggregatable           bool                   `json:"aggregatable"`
+	Indices                []string               `json:"indices,omitempty"`
+	NonSearchableIndices   []string               `json:"non_searchable_indices,omitempty"`
+	NonAggregatableIndices []string               `json:"non_aggregatable_indices,omitempty"`
+	Meta                   map[string]interface{} `json:"meta,omitempty"`
 }

field_caps_test.go

@@ -96,25 +96,25 @@ func TestFieldCapsResponse(t *testing.T) {
 				"failed": 0
 		},
 		"fields": {
-			"rating": { 
+			"rating": {
 				"long": {
 					"searchable": true,
 					"aggregatable": false,
 					"indices": ["index1", "index2"],
-					"non_aggregatable_indices": ["index1"] 
+					"non_aggregatable_indices": ["index1"]
 				},
 				"keyword": {
 					"searchable": false,
 					"aggregatable": true,
 					"indices": ["index3", "index4"],
-					"non_searchable_indices": ["index4"] 
+					"non_searchable_indices": ["index4"]
 				}
 			},
-			"title": { 
+			"title": {
 				"text": {
 					"searchable": true,
 					"aggregatable": false
-	
+
 				}
 			}
 		}

runtime_mappings.go

@@ -0,0 +1,16 @@
+// Copyright 2012-present Oliver Eilhard. All rights reserved.
+// Use of this source code is governed by a MIT-license.
+// See http://olivere.mit-license.org/license.txt for details.
+
+package elastic
+
+// RuntimeMappings specify fields that are evaluated at query time.
+//
+// See https://www.elastic.co/guide/en/elasticsearch/reference/7.14/runtime.html
+// for details.
+type RuntimeMappings map[string]interface{}
+
+// Source deserializes the runtime mappings.
+func (m *RuntimeMappings) Source() (interface{}, error) {
+	return m, nil
+}

runtime_mappings_test.go

@@ -0,0 +1,148 @@
+// Copyright 2012-present Oliver Eilhard. All rights reserved.
+// Use of this source code is governed by a MIT-license.
+// See http://olivere.mit-license.org/license.txt for details.
+
+package elastic
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+func TestRuntimeMappingsSource(t *testing.T) {
+	rm := RuntimeMappings{
+		"day_of_week": map[string]interface{}{
+			"type": "keyword",
+		},
+	}
+	src, err := rm.Source()
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := json.Marshal(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expected := `{"day_of_week":{"type":"keyword"}}`
+	if want, have := expected, string(data); want != have {
+		t.Fatalf("want %s, have %s", want, have)
+	}
+}
+
+func TestRuntimeMappings(t *testing.T) {
+	client := setupTestClient(t) //, SetTraceLog(log.New(os.Stdout, "", 0)))
+
+	ctx := context.Background()
+	indexName := testIndexName
+
+	// Create index
+	createIndex, err := client.CreateIndex(indexName).Do(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if createIndex == nil {
+		t.Errorf("expected result to be != nil; got: %v", createIndex)
+	}
+
+	mapping := `{
+		"dynamic": "runtime",
+		"properties": {
+			"@timestamp": {
+				"type":"date"
+			}
+		},
+		"runtime": {
+			"day_of_week": {
+				"type": "keyword",
+				"script": {
+					"source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+				}
+			}
+		}
+	}`
+	type Doc struct {
+		Timestamp time.Time `json:"@timestamp"`
+	}
+	type DynamicDoc struct {
+		Timestamp time.Time `json:"@timestamp"`
+		DayOfWeek string    `json:"day_of_week"`
+	}
+
+	// Create mapping
+	putResp, err := client.PutMapping().
+		Index(indexName).
+		BodyString(mapping).
+		Do(ctx)
+	if err != nil {
+		t.Fatalf("expected put mapping to succeed; got: %v", err)
+	}
+	if putResp == nil {
+		t.Fatalf("expected put mapping response; got: %v", putResp)
+	}
+	if !putResp.Acknowledged {
+		t.Fatalf("expected put mapping ack; got: %v", putResp.Acknowledged)
+	}
+
+	// Add a document
+	timestamp := time.Date(2021, 1, 17, 23, 24, 25, 26, time.UTC)
+	indexResult, err := client.Index().
+		Index(indexName).
+		Id("1").
+		BodyJson(&Doc{
+			Timestamp: timestamp,
+		}).
+		Refresh("wait_for").
+		Do(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if indexResult == nil {
+		t.Errorf("expected result to be != nil; got: %v", indexResult)
+	}
+
+	// Execute a search to check for runtime fields
+	searchResp, err := client.Search(indexName).
+		Query(NewMatchAllQuery()).
+		DocvalueFields("@timestamp", "day_of_week").
+		Do(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if searchResp == nil {
+		t.Errorf("expected result to be != nil; got: %v", searchResp)
+	}
+	if want, have := int64(1), searchResp.TotalHits(); want != have {
+		t.Fatalf("expected %d search hits, got %d", want, have)
+	}
+
+	// The hit should not have the "day_of_week"
+	hit := searchResp.Hits.Hits[0]
+	var doc DynamicDoc
+	if err := json.Unmarshal(hit.Source, &doc); err != nil {
+		t.Fatalf("unable to deserialize hit: %v", err)
+	}
+	if want, have := timestamp, doc.Timestamp; want != have {
+		t.Fatalf("expected timestamp=%v, got %v", want, have)
+	}
+	if want, have := "", doc.DayOfWeek; want != have {
+		t.Fatalf("expected day_of_week=%q, got %q", want, have)
+	}
+
+	// The fields should include a "day_of_week" of ["Sunday"]
+	dayOfWeekIntfSlice, ok := hit.Fields["day_of_week"].([]interface{})
+	if !ok {
+		t.Fatalf("expected a slice of strings, got %T", hit.Fields["day_of_week"])
+	}
+	if want, have := 1, len(dayOfWeekIntfSlice); want != have {
+		t.Fatalf("expected a slice of size %d, have %d", want, have)
+	}
+	dayOfWeek, ok := dayOfWeekIntfSlice[0].(string)
+	if !ok {
+		t.Fatalf("expected an element of string, got %T", dayOfWeekIntfSlice[0])
+	}
+	if want, have := "Sunday", dayOfWeek; want != have {
+		t.Fatalf("expected day_of_week=%q, have %q", want, have)
+	}
+}

search.go

@@ -159,6 +159,12 @@ func (s *SearchService) PointInTime(pointInTime *PointInTime) *SearchService {
 	return s
 }
 
+// RuntimeMappings specifies optional runtime mappings.
+func (s *SearchService) RuntimeMappings(runtimeMappings RuntimeMappings) *SearchService {
+	s.searchSource = s.searchSource.RuntimeMappings(runtimeMappings)
+	return s
+}
+
 // TimeoutInMillis sets the timeout in milliseconds.
 func (s *SearchService) TimeoutInMillis(timeoutInMillis int) *SearchService {
 	s.searchSource = s.searchSource.TimeoutInMillis(timeoutInMillis)
@@ -764,7 +770,7 @@ type SearchHit struct {
 	Sort           []interface{}                  `json:"sort,omitempty"`            // sort information
 	Highlight      SearchHitHighlight             `json:"highlight,omitempty"`       // highlighter information
 	Source         json.RawMessage                `json:"_source,omitempty"`         // stored document source
-	Fields         map[string]interface{}         `json:"fields,omitempty"`          // returned (stored) fields
+	Fields         SearchHitFields                `json:"fields,omitempty"`          // returned (stored) fields
 	Explanation    *SearchExplanation             `json:"_explanation,omitempty"`    // explains how the score was computed
 	MatchedQueries []string                       `json:"matched_queries,omitempty"` // matched queries
 	InnerHits      map[string]*SearchHitInnerHits `json:"inner_hits,omitempty"`      // inner hits with ES >= 1.5.0
@@ -777,6 +783,43 @@ type SearchHit struct {
 	// MatchedFilters
 }
 
+// SearchHitFields helps to simplify resolving slices of specific types.
+type SearchHitFields map[string]interface{}
+
+// Strings returns a slice of strings for the given field, if there is any
+// such field in the hit. The method ignores elements that are not of type
+// string.
+func (f SearchHitFields) Strings(fieldName string) ([]string, bool) {
+	slice, ok := f[fieldName].([]interface{})
+	if !ok {
+		return nil, false
+	}
+	results := make([]string, 0, len(slice))
+	for _, item := range slice {
+		if v, ok := item.(string); ok {
+			results = append(results, v)
+		}
+	}
+	return results, true
+}
+
+// Float64s returns a slice of float64's for the given field, if there is any
+// such field in the hit. The method ignores elements that are not of
+// type float64.
+func (f SearchHitFields) Float64s(fieldName string) ([]float64, bool) {
+	slice, ok := f[fieldName].([]interface{})
+	if !ok {
+		return nil, false
+	}
+	results := make([]float64, 0, len(slice))
+	for _, item := range slice {
+		if v, ok := item.(float64); ok {
+			results = append(results, v)
+		}
+	}
+	return results, true
+}
+
 // SearchHitInnerHits is used for inner hits.
 type SearchHitInnerHits struct {
 	Hits *SearchHits `json:"hits,omitempty"`