Merge branch 'upstream/main' into number-support-single-type

* upstream/main: (265 commits)
  Disable openid connect tests due to missing fixture (elastic#89478)
  Add periodic job for single processor node testing
  Updates to changelog processing after docs redesign (elastic#89463)
  Better support for multi cluster for run task (elastic#89442)
  Mute failing tests (elastic#89465)
  [ML] Performance improvements related to ECS Grok pattern usage (elastic#89424)
  Add source fallback support for date and date_nanos mapped types (elastic#89440)
  Reuse Info in lifecycle step (elastic#89419)
  feature: support metrics for multi value fields (elastic#88818)
  Upgrade OpenTelemetry API and remove workaround (elastic#89438)
  Remove LegacyClusterTaskResultActionListener (elastic#89459)
  Add YAML spec docs about matching errors (elastic#89370)
  Remove redundant cluster upgrade tests for auth tokens (elastic#89417)
  Return 400 error for GetUserPrivileges call with API keys (elastic#89333)
  User Profile - Detailed errors in hasPrivileges response (elastic#89224)
  Rollover min_* conditions docs and highlight (elastic#89434)
  REST tests for percentiles_bucket agg (elastic#88029)
  REST tests for cumulative pipeline aggs (elastic#88966)
  Clean-up file watcher keys. (elastic#89429)
  fix a typo in Security.java (elastic#89248)
  ...

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -14,7 +14,8 @@
           name: os
           values:
             - "centos-7&&immutable"
-            - "amazon&&immutable"
+            - "amazon-2&&immutable"
+            - "amazon-2022&&immutable"
             - "debian-10&&immutable"
             - "debian-11&&immutable"
             - "opensuse-15-1&&immutable"

.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml

@@ -22,8 +22,8 @@
           export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
 
           mkdir -p ${BEATS_DIR}
-          curl -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
-          curl -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
 
           $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false \
             -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build

.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-node-tests.yml

@@ -0,0 +1,16 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests
+    display-name: "elastic / elasticsearch # %BRANCH% - single processor node tests"
+    description: "Testing with node.processors set to '1' for the Elasticsearch %BRANCH% branch.\n"
+    node: "general-purpose && docker"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA11_HOME=$HOME/.java/java11
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true check

.ci/jobs.t/elastic+elasticsearch+periodic+single-processor-tests-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+periodic+single-processor-node-tests
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml

@@ -31,7 +31,7 @@
           type: label-expression
           name: os
           values:
-            - rocky-linux-8-packaging
+            - rhel-8-packaging
             - ubuntu-20.04-packaging
       - axis:
           type: user-defined

.ci/scripts/packaging-test.sh

@@ -43,6 +43,13 @@ if [ -f "/etc/os-release" ] ; then
             sudo apt-get install -y --allow-downgrades lintian=2.15.0
         fi
     fi
+    if [[ "$ID" == "rhel" ]] ; then
+      # Downgrade containerd if necessary to work around runc bug
+      # See: https://github.com/opencontainers/runc/issues/3551
+      if containerd -version | grep -sF 1.6.7; then
+        sudo yum downgrade -y containerd.io
+      fi
+    fi
 else
     cat /etc/issue || true
 fi

.github/CODEOWNERS

@@ -15,3 +15,7 @@ x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json @elastic/infra
 x-pack/plugin/core/src/main/resources/monitoring-logstash.json @elastic/infra-monitoring-ui
 x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json @elastic/infra-monitoring-ui
 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/infra-monitoring-ui
+
+# Elastic Agent
+x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/elastic-agent-control-plane
+x-pack/plugin/core/src/main/resources/fleet-* @elastic/elastic-agent-control-plane

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,8 +7,8 @@ attention.
 -->
 
 - Have you signed the [contributor license agreement](https://www.elastic.co/contributor-agreement)?
-- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md)?
+- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/main/CONTRIBUTING.md)?
 - If submitting code, have you built your formula locally prior to submission with `gradle check`?
-- If submitting code, is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed.
+- If submitting code, is your pull request against main? Unless there is a good reason otherwise, we prefer pull requests against main and will backport as needed.
 - If submitting code, have you checked that your submission is for an [OS and architecture that we support](https://www.elastic.co/support/matrix#show_os)?
-- If you are submitting this code for a class then read our [policy](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md#contributing-as-part-of-a-class) for that.
+- If you are submitting this code for a class then read our [policy](https://github.com/elastic/elasticsearch/blob/main/CONTRIBUTING.md#contributing-as-part-of-a-class) for that.

BUILDING.md

@@ -63,6 +63,38 @@ E.g. [configuration-cache support](https://github.com/elastic/elasticsearch/issu
 There are a few guidelines to follow that should make your life easier to make changes to the elasticsearch build.
 Please add a member of the `es-delivery` team as a reviewer if you're making non-trivial changes to the build.
 
+#### Adding or updating a dependency
+
+We rely on [Gradle dependency verification](https://docs.gradle.org/current/userguide/dependency_verification.html) to mitigate the security risks and avoid integrating compromised dependencies.
+
+This requires to have third party dependencies and their checksums listed in `gradle/verification-metadata.xml`.
+
+For updated or newly added dependencies you need to add an entry to this verification file or update the existing one:
+```
+      <component group="asm" name="asm" version="3.1">
+         <artifact name="asm-3.1.jar">
+            <sha256 value="333ff5369043975b7e031b8b27206937441854738e038c1f47f98d072a20437a" origin="official site"/>
+         </artifact>
+      </component>
+```
+
+You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
+```
+>./gradlew --write-verification-metadata sha256 precommit
+```
+
+The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations, 
+but we use detached configurations for a certain set of plugins and tasks. Therefore, please ensure you run this option with a task that
+uses the changed dependencies. In most cases, `precommit` or `check` are good candidates.
+
+We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry 
+will have the `origin` attribute been set to `Generated by Gradle`. 
+
+>A manual confirmation of the Gradle generated checksums is currently not mandatory.
+>If you want to add a level of verification you can manually confirm the checksum (e.g by looking it up on the website of the library)
+>Please replace the content of the `origin` attribute by `official site` in that case.
+>
+
 #### Custom Plugin and Task implementations
 
 Build logic that is used across multiple subprojects should considered to be moved into a Gradle plugin with according Gradle task implmentation.

CHANGELOG.md

@@ -1,3 +1,3 @@
-# Elasticsearch Changlog
+# Elasticsearch Changelog
 
 Please see the [release notes](https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html) in the reference manual.

TRACING.md

@@ -0,0 +1,156 @@
+# Tracing in Elasticsearch
+
+Elasticsearch is instrumented using the [OpenTelemetry][otel] API, which allows
+us to gather traces and analyze what Elasticsearch is doing.
+
+
+## How is tracing implemented?
+
+The Elasticsearch server code contains a [`tracing`][tracing] package, which is
+an abstraction over the OpenTelemetry API. All locations in the code that
+perform instrumentation and tracing must use these abstractions.
+
+Separately, there is the [`apm`](./modules/apm/) module, which works with the
+OpenTelemetry API directly to record trace data.  Underneath the OTel API, we
+use Elastic's [APM agent for Java][agent], which attaches at runtime to the
+Elasticsearch JVM and removes the need for Elasticsearch to hard-code the use of
+an OTel implementation. Note that while it is possible to programmatically start
+the APM agent, the Security Manager permissions required make this essentially
+impossible.
+
+
+## How is tracing configured?
+
+You must supply configuration and credentials for the APM server (see below).
+You must also set `tracing.apm.enabled` to `true`, but this can be toggled at
+runtime.
+
+All APM settings live under `tracing.apm`. All settings related to the Java agent
+go under `tracing.apm.agent`. Anything you set under there will be propagated to
+the agent.
+
+For agent settings that can be changed dynamically, you can use the cluster
+settings REST API. For example, to change the sampling rate:
+
+    curl -XPUT \
+      -H "Content-type: application/json" \
+      -u "$USERNAME:$PASSWORD" \
+      -d '{ "persistent": { "tracing.apm.agent.transaction_sample_rate": "0.75" } }' \
+      https://localhost:9200/_cluster/settings
+
+
+### More details about configuration
+
+For context, the APM agent pulls configuration from [multiple
+sources][agent-config], with a hierarchy that means, for example, that options
+set in the config file cannot be overridden via system properties.
+
+Now, in order to send tracing data to the APM server, ES needs to be configured with
+either a `secret_key` or an `api_key`. We could configure these in the agent via
+system properties, but then their values would be available to any Java code in
+Elasticsearch that can read system properties.
+
+Instead, when Elasticsearch bootstraps itself, it compiles all APM settings
+together, including any `secret_key` or `api_key` values from the ES keystore,
+and writes out a temporary APM config file containing all static configuration
+(i.e. values that cannot change after the agent starts).  This file is deleted
+as soon as possible after ES starts up. Settings that are not sensitive and can
+be changed dynamically are configured via system properties. Calls to the ES
+settings REST API are translated into system property writes, which the agent
+later picks up and applies.
+
+## Where is tracing data sent?
+
+You need to have an APM server running somewhere. For example, you can create a
+deployment in [Elastic Cloud](https://www.elastic.co/cloud/) with Elastic's APM
+integration.
+
+## What do we trace?
+
+We primarily trace "tasks". The tasks framework in Elasticsearch allows work to
+be scheduled for execution, cancelled, executed in a different thread pool, and
+so on. Tracing a task results in a "span", which represents the execution of the
+task in the tracing system. We also instrument REST requests, which are not (at
+present) modelled by tasks.
+
+A span can be associated with a parent span, which allows all spans in, for
+example, a REST request to be grouped together. Spans can track work across
+different Elasticsearch nodes.
+
+Elasticsearch also supports distributed tracing via [W3c Trace Context][w3c]
+headers. If clients of Elasticsearch send these headers with their requests,
+then that data will be forwarded to the APM server in order to yield a trace
+across systems.
+
+In rare circumstances, it is possible to avoid tracing a task using
+`TaskManager#register(String,String,TaskAwareRequest,boolean)`. For example,
+Machine Learning uses tasks to record which models are loaded on each node. Such
+tasks are long-lived and are not suitable candidates for APM tracing.
+
+## Thread contexts and nested spans
+
+When a span is started, Elasticsearch tracks information about that span in the
+current [thread context][thread-context].  If a new thread context is created,
+then the current span information must not propagated but instead renamed, so
+that (1) it doesn't interfere when new trace information is set in the context,
+and (2) the previous trace information is available to establish a parent /
+child span relationship.  This is done with `ThreadContext#newTraceContext()`.
+
+Sometimes we need to detach new spans from their parent. For example, creating
+an index starts some related background tasks, but these shouldn't be associated
+with the REST request, otherwise all the background task spans will be
+associated with the REST request for as long as Elasticsearch is running.
+`ThreadContext` provides the `clearTraceContext`() method for this purpose.
+
+## How to I trace something that isn't a task?
+
+First work out if you can turn it into a task. No, really.
+
+If you can't do that, you'll need to ensure that your class can get access to a
+`Tracer` instance (this is available to inject, or you'll need to pass it when
+your class is created). Then you need to call the appropriate methods on the
+tracer when a span should start and end. You'll also need to manage the creation
+of new trace contexts when child spans need to be created.
+
+## What additional attributes should I set?
+
+That's up to you. Be careful not to capture anything that could leak sensitive
+or personal information.
+
+## What is "scope" and when should I used it?
+
+Usually you won't need to.
+
+That said, sometimes you may want more details to be captured about a particular
+section of code. You can think of "scope" as representing the currently active
+tracing context. Using scope allows the APM agent to do the following:
+
+* Enables automatic correlation between the "active span" and logging, where
+  logs have also been captured.
+* Enables capturing any exceptions thrown when the span is active, and linking
+  those exceptions to the span
+* Allows the sampling profiler to be used as it allows samples to be linked to
+  the active span (if any), so the agent can automatically get extra spans
+  without manual instrumentation.
+
+However, a scope must be closed in the same thread in which it was opened, which
+cannot be guaranteed when using tasks, making scope largely useless to
+Elasticsearch.
+
+In the OpenTelemetry documentation, spans, scope and context are fairly
+straightforward to use, since `Scope` is an `AutoCloseable` and so can be
+easily created and cleaned up use try-with-resources blocks. Unfortunately,
+Elasticsearch is a complex piece of software, and also extremely asynchronous,
+so the typical OpenTelemetry examples do not work.
+
+Nonetheless, it is possible to manually use scope where we need more detail by
+explicitly opening a scope via the `Tracer`.
+
+
+[otel]: https://opentelemetry.io/
+[thread-context]: ./server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java).
+[w3c]: https://www.w3.org/TR/trace-context/
+[tracing]: ./server/src/main/java/org/elasticsearch/tracing/
+[config]: ./modules/apm/src/main/config/elasticapm.properties
+[agent-config]: https://www.elastic.co/guide/en/apm/agent/java/master/configuration.html
+[agent]: https://www.elastic.co/guide/en/apm/agent/java/current/index.html

benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java

@@ -8,8 +8,6 @@
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.FetchSourcePhase;
 import org.elasticsearch.search.lookup.SourceLookup;
-import org.elasticsearch.xcontent.DeprecationHandler;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -108,8 +106,7 @@ public BytesReference filterXContentOnBuilder() throws IOException {
             XContentType.JSON.toParsedMediaType()
         );
         try (
-            XContentParser parser = XContentType.JSON.xContent()
-                .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, sourceBytes.streamInput())
+            XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, sourceBytes.streamInput())
         ) {
             builder.copyCurrentStructure(parser);
             return BytesReference.bytes(builder);

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=97a52d145762adc241bad7fd18289bf7f6801e08ece6badf80402fe2b9f250b1
+distributionSha256Sum=db9c8211ed63f61f60292c69e80d89196f9eb36665e369e7f00ac4cc841c2219

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -39,7 +39,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         """
 
         when:
-        def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
+        def result = gradleRunner("setupDistro", '-g', gradleUserHome).build()
 
         then:
         result.task(":distribution:archives:${testArchiveProjectName}:buildExpanded").outcome == TaskOutcome.SUCCESS

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.gradle.internal
 
+import spock.lang.TempDir
 import spock.lang.Unroll
 import com.github.tomakehurst.wiremock.WireMockServer
 
@@ -126,7 +127,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         when:
         def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
             buildFile << repositoryMockSetup(server, jdkVendor, jdkVersion)
-            gradleRunner('getJdk', '-i', '-g', testProjectDir.newFolder().toString()).build()
+            gradleRunner('getJdk', '-i', '-g', gradleUserHome).build()
         }
 
         then:
@@ -179,13 +180,12 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
             buildFile << repositoryMockSetup(server, VENDOR_ADOPTIUM, ADOPT_JDK_VERSION)
 
-            def commonGradleUserHome = testProjectDir.newFolder().toString()
             // initial run
-            def firstResult = gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', commonGradleUserHome).build()
+            def firstResult = gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', gradleUserHome).build()
             // assert the output of an executed transform is shown
             assertOutputContains(firstResult.output, "Unpacking $expectedArchiveName using $transformType")
             // run against up-to-date transformations
-            gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', commonGradleUserHome).build()
+            gradleRunner('clean', 'getJdk', '-i', '--warning-mode', 'all', '-g', gradleUserHome).build()
         }
 
         then:

build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle

@@ -23,7 +23,6 @@ if (buildNumber && performanceTest == null) {
         fileset(dir: projectDir) {
           Set<File> fileSet = fileTree(projectDir) {
             include("**/*.hprof")
-            include("**/reaper.log")
             include("**/build/test-results/**/*.xml")
             include("**/build/testclusters/**")
             exclude("**/build/testclusters/**/data/**")
@@ -49,6 +48,8 @@ if (buildNumber && performanceTest == null) {
         }
 
         fileset(dir: "${gradle.gradleUserHomeDir}/workers", followsymlinks: false)
+
+        fileset(dir: "${project.projectDir}/.gradle/reaper", followsymlinks: false)
       }
     } catch (Exception e) {
       logger.lifecycle("Failed to archive additional logs", e)