diff --git a/distribution/src/bin/elasticsearch b/distribution/src/bin/elasticsearch index 178e490c80dcb..a4bcde1a92af6 100755 --- a/distribution/src/bin/elasticsearch +++ b/distribution/src/bin/elasticsearch @@ -47,6 +47,11 @@ if [ -z "$ES_TMPDIR" ]; then ES_TMPDIR=`"$JAVA" "$XSHARE" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory` fi +if [ -z "$LIBFFI_TMPDIR" ]; then + LIBFFI_TMPDIR="$ES_TMPDIR" + export LIBFFI_TMPDIR +fi + # get keystore password before setting java options to avoid # conflicting GC configurations for the keystore tools unset KEYSTORE_PASSWORD diff --git a/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc b/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc index da3ccc1c50a30..2fc52cb098146 100644 --- a/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc +++ b/docs/reference/setup/sysconfig/executable-jna-tmpdir.asciidoc @@ -4,22 +4,34 @@ [NOTE] This is only relevant for Linux. -Elasticsearch uses the Java Native Access (JNA) library for executing some -platform-dependent native code. On Linux, the native code backing this library -is extracted at runtime from the JNA archive. This code is extracted -to the Elasticsearch temporary directory which defaults to a sub-directory of -`/tmp` and can be configured with the <> variable. -Alternatively, this location can be controlled with the JVM flag -`-Djna.tmpdir=`. As the native library is mapped into the JVM virtual -address space as executable, the underlying mount point of the location that -this code is extracted to must *not* be mounted with `noexec` as this prevents -the JVM process from being able to map this code as executable. On some hardened -Linux installations this is a default mount option for `/tmp`. One indication -that the underlying mount is mounted with `noexec` is that at startup JNA will -fail to load with a `java.lang.UnsatisfiedLinkerError` exception with a message -along the lines of `failed to map segment from shared object`. Note that the -exception message can differ amongst JVM versions. Additionally, the components -of Elasticsearch that rely on execution of native code via JNA will fail with -messages indicating that it is `because JNA is not available`. If you are seeing -such error messages, you must remount the temporary directory used for JNA to -not be mounted with `noexec`. +{es} uses the Java Native Access (JNA) library, and another library called +`libffi`, for executing some platform-dependent native code. On Linux, the +native code backing these libraries is extracted at runtime into a temporary +directory and then mapped into executable pages in {es}'s address space. This +requires the underlying files not to be on a filesystem mounted with the +`noexec` option. + +By default, {es} will create its temporary directory within `/tmp`. However, +some hardened Linux installations mount `/tmp` with the `noexec` option by +default. This prevents JNA and `libffi` from working correctly. For instance, +at startup JNA may fail to load with an `java.lang.UnsatisfiedLinkerError` +exception or with a message along the lines of `failed to map segment from +shared object`. Note that the exception message can differ amongst JVM +versions. Additionally, the components of {es} that rely on execution of native +code via JNA may fail with messages indicating that it is `because JNA is not +available`. + +To resolve these problems, either remove the `noexec` option from your `/tmp` +filesystem, or configure {es} to use a different location for its temporary +directory by setting the <> environment variable. For +instance: + +["source","sh",subs="attributes"] +-------------------------------------------- +export ES_TMPDIR=/usr/share/elasticsearch/tmp +-------------------------------------------- + +Alternatively, you can configure the path that JNA uses for its temporary files +with the <> `-Djna.tmpdir=` and you can +configure the path that `libffi` uses for its temporary files with the +`LIBFFI_TMPDIR` environment variable. diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/TemporaryDirectoryConfigTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/TemporaryDirectoryConfigTests.java new file mode 100644 index 0000000000000..541458d201005 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/TemporaryDirectoryConfigTests.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.packaging.test; + +import org.apache.http.client.fluent.Request; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.ServerUtils; +import org.elasticsearch.packaging.util.Shell; +import org.elasticsearch.packaging.util.docker.DockerRun; +import org.junit.After; +import org.junit.Before; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.elasticsearch.packaging.util.docker.Docker.removeContainer; +import static org.elasticsearch.packaging.util.docker.Docker.runContainer; +import static org.elasticsearch.packaging.util.docker.Docker.runContainerExpectingFailure; +import static org.elasticsearch.packaging.util.docker.Docker.waitForElasticsearch; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; + +public class TemporaryDirectoryConfigTests extends PackagingTestCase { + + @Before + public void onlyLinux() { + assumeTrue("only Linux", distribution.platform == Distribution.Platform.LINUX); + } + + @After + public void cleanupContainer() { + if (distribution().isDocker()) { + removeContainer(); + } + } + + public void test10Install() throws Exception { + install(); + setFileSuperuser("test_superuser", "test_superuser_password"); + } + + public void test20AcceptsCustomPath() throws Exception { + assumeFalse(distribution().isDocker()); + + final Path tmpDir = createTempDir("libffi"); + sh.getEnv().put("LIBFFI_TMPDIR", tmpDir.toString()); + withLibffiTmpdir(tmpDir.toString(), confPath -> assertWhileRunning(() -> { + ServerUtils.makeRequest( + Request.Get("https://localhost:9200/"), + "test_superuser", + "test_superuser_password", + ServerUtils.getCaCert(confPath) + ); // just checking it doesn't throw + })); + } + + public void test21AcceptsCustomPathInDocker() throws Exception { + assumeTrue(distribution().isDocker()); + + final Path tmpDir = createTempDir("libffi"); + + installation = runContainer( + distribution(), + DockerRun.builder() + // There's no actual need for this to be a bind-mounted dir, but it's the quickest + // way to create a directory in the container before the entrypoint runs. + .volume(tmpDir, tmpDir) + .envVar("ELASTIC_PASSWORD", "nothunter2") + .envVar("LIBFFI_TMPDIR", tmpDir.toString()) + ); + + waitForElasticsearch(installation, "elastic", "nothunter2"); + } + + public void test30VerifiesCustomPath() throws Exception { + assumeFalse(distribution().isDocker()); + + final Path tmpFile = createTempDir("libffi").resolve("file"); + Files.createFile(tmpFile); + withLibffiTmpdir( + tmpFile.toString(), + confPath -> assertElasticsearchFailure(runElasticsearchStartCommand(null, false, false), "LIBFFI_TMPDIR", null) + ); + } + + public void test31VerifiesCustomPathInDocker() throws Exception { + assumeTrue(distribution().isDocker()); + + final Path tmpDir = createTempDir("libffi"); + final Path tmpFile = tmpDir.resolve("file"); + Files.createFile(tmpFile); + + final Shell.Result result = runContainerExpectingFailure( + distribution(), + DockerRun.builder().volume(tmpDir, tmpDir).envVar("LIBFFI_TMPDIR", tmpFile.toString()) + ); + assertThat(result.stderr, containsString("LIBFFI_TMPDIR")); + } + + private void withLibffiTmpdir(String tmpDir, CheckedConsumer action) throws Exception { + sh.getEnv().put("LIBFFI_TMPDIR", tmpDir); + withCustomConfig(confPath -> { + if (distribution.isPackage()) { + append(installation.envFile, "LIBFFI_TMPDIR=" + tmpDir); + } + action.accept(confPath); + }); + + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index 668c1fc416394..8a81828751dd0 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -204,7 +204,7 @@ private static void waitForElasticsearchToExit() { } catch (Exception e) { logger.warn("Caught exception while waiting for ES to exit", e); } - } while (attempt++ < 8); + } while (attempt++ < 60); if (isElasticsearchRunning) { final Shell.Result dockerLogs = getContainerLogs(); diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index 7bde9dacc572e..0fb73957a9ec0 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -173,6 +173,11 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot throw new BootstrapException(e); } + try { + environment.validateNativesConfig(); // temporary directories are important for JNA + } catch (IOException e) { + throw new BootstrapException(e); + } initializeNatives( environment.tmpFile(), BootstrapSettings.MEMORY_LOCK_SETTING.get(settings), diff --git a/server/src/main/java/org/elasticsearch/env/Environment.java b/server/src/main/java/org/elasticsearch/env/Environment.java index f4d843885e22d..13acbf591e50e 100644 --- a/server/src/main/java/org/elasticsearch/env/Environment.java +++ b/server/src/main/java/org/elasticsearch/env/Environment.java @@ -8,6 +8,7 @@ package org.elasticsearch.env; +import org.apache.lucene.util.Constants; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -304,12 +305,44 @@ public Path tmpFile() { /** Ensure the configured temp directory is a valid directory */ public void validateTmpFile() throws IOException { - if (Files.exists(tmpFile) == false) { - throw new FileNotFoundException("Temporary file directory [" + tmpFile + "] does not exist or is not accessible"); + validateTemporaryDirectory("Temporary directory", tmpFile); + } + + /** + * Ensure the temp directories needed for JNA are set up correctly. + */ + public void validateNativesConfig() throws IOException { + validateTmpFile(); + if (Constants.LINUX) { + validateTemporaryDirectory(LIBFFI_TMPDIR_ENVIRONMENT_VARIABLE + " environment variable", getLibffiTemporaryDirectory()); + } + } + + private static void validateTemporaryDirectory(String description, Path path) throws IOException { + if (path == null) { + throw new NullPointerException(description + " was not specified"); + } + if (Files.exists(path) == false) { + throw new FileNotFoundException(description + " [" + path + "] does not exist or is not accessible"); } - if (Files.isDirectory(tmpFile) == false) { - throw new IOException("Configured temporary file directory [" + tmpFile + "] is not a directory"); + if (Files.isDirectory(path) == false) { + throw new IOException(description + " [" + path + "] is not a directory"); + } + } + + private static final String LIBFFI_TMPDIR_ENVIRONMENT_VARIABLE = "LIBFFI_TMPDIR"; + + @SuppressForbidden(reason = "using PathUtils#get since libffi resolves paths without interference from the JVM") + private static Path getLibffiTemporaryDirectory() { + final String workingDirectory = System.getProperty("user.dir"); + if (workingDirectory == null) { + return null; + } + final String environmentVariable = System.getenv(LIBFFI_TMPDIR_ENVIRONMENT_VARIABLE); + if (environmentVariable == null) { + return null; } + return PathUtils.get(workingDirectory).resolve(environmentVariable); } /** Returns true if the data path is a list, false otherwise */ diff --git a/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java index 386fba636c0f8..68bcaac0e401b 100644 --- a/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/EnvironmentTests.java @@ -116,7 +116,7 @@ public void testNonExistentTempPathValidation() { Settings build = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); Environment environment = new Environment(build, null, createTempDir().resolve("this_does_not_exist")); FileNotFoundException e = expectThrows(FileNotFoundException.class, environment::validateTmpFile); - assertThat(e.getMessage(), startsWith("Temporary file directory [")); + assertThat(e.getMessage(), startsWith("Temporary directory [")); assertThat(e.getMessage(), endsWith("this_does_not_exist] does not exist or is not accessible")); } @@ -124,7 +124,23 @@ public void testTempPathValidationWhenRegularFile() throws IOException { Settings build = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); Environment environment = new Environment(build, null, createTempFile("something", ".test")); IOException e = expectThrows(IOException.class, environment::validateTmpFile); - assertThat(e.getMessage(), startsWith("Configured temporary file directory [")); + assertThat(e.getMessage(), startsWith("Temporary directory [")); + assertThat(e.getMessage(), endsWith(".test] is not a directory")); + } + + public void testNonExistentTempPathValidationForNatives() { + Settings build = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + Environment environment = new Environment(build, null, createTempDir().resolve("this_does_not_exist")); + FileNotFoundException e = expectThrows(FileNotFoundException.class, environment::validateNativesConfig); + assertThat(e.getMessage(), startsWith("Temporary directory [")); + assertThat(e.getMessage(), endsWith("this_does_not_exist] does not exist or is not accessible")); + } + + public void testTempPathValidationWhenRegularFileForNatives() throws IOException { + Settings build = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + Environment environment = new Environment(build, null, createTempFile("something", ".test")); + IOException e = expectThrows(IOException.class, environment::validateNativesConfig); + assertThat(e.getMessage(), startsWith("Temporary directory [")); assertThat(e.getMessage(), endsWith(".test] is not a directory")); }