From d5781db7a00f2c73f60597b233986fdf3d0ab4bb Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 21 Feb 2022 09:54:54 +0100 Subject: [PATCH 1/3] Elasticsearch: Ensure Elasticsearch 8 works OOTB secure as default Since Elasticsearch 8.0 the default is to enable security, meaning TLS and authentication. This adds a check for Elasticsearch 8.0 to change the default behaviour to properly support this change, but you can still run Elasticsearch with security features disabled, if you want. --- .../elasticsearch/ElasticsearchContainer.java | 106 +++++++++++++++--- .../ElasticsearchContainerTest.java | 40 ++++++- 2 files changed, 124 insertions(+), 22 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java index acbca15aaf5..f95b904a597 100644 --- a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java +++ b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java @@ -1,21 +1,38 @@ package org.testcontainers.elasticsearch; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; - -import java.net.InetSocketAddress; -import java.time.Duration; +import com.github.dockerjava.api.command.InspectContainerResponse; +import org.apache.commons.io.IOUtils; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.Base58; import org.testcontainers.utility.DockerImageName; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import java.io.ByteArrayInputStream; +import java.net.InetSocketAddress; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.time.Duration; +import java.util.Optional; +import java.util.regex.Pattern; + +import static java.net.HttpURLConnection.HTTP_OK; +import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; + /** * Represents an elasticsearch docker instance which exposes by default port 9200 and 9300 (transport.tcp.port) * The docker image is by default fetched from docker.elastic.co/elasticsearch/elasticsearch */ public class ElasticsearchContainer extends GenericContainer { + /** + * Elasticsearch Default Password for Elasticsearch >= 8 + */ + public static final String ELASTICSEARCH_DEFAULT_PASSWORD = "changeme"; + /** * Elasticsearch Default HTTP port */ @@ -39,7 +56,13 @@ public class ElasticsearchContainer extends GenericContainer caCertAsBytes = Optional.empty(); /** * @deprecated use {@link ElasticsearchContainer(DockerImageName)} instead @@ -65,23 +88,71 @@ public ElasticsearchContainer(final DockerImageName dockerImageName) { super(dockerImageName); dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME, DEFAULT_OSS_IMAGE_NAME); - - if (dockerImageName.isCompatibleWith(DEFAULT_OSS_IMAGE_NAME)) { - this.isOss = true; - } + this.isOss = dockerImageName.isCompatibleWith(DEFAULT_OSS_IMAGE_NAME); logger().info("Starting an elasticsearch container using [{}]", dockerImageName); withNetworkAliases("elasticsearch-" + Base58.randomString(6)); withEnv("discovery.type", "single-node"); addExposedPorts(ELASTICSEARCH_DEFAULT_PORT, ELASTICSEARCH_DEFAULT_TCP_PORT); - setWaitStrategy(new HttpWaitStrategy() - .forPort(ELASTICSEARCH_DEFAULT_PORT) - .forStatusCodeMatching(response -> response == HTTP_OK || response == HTTP_UNAUTHORIZED) - .withStartupTimeout(Duration.ofMinutes(2))); + this.isAtLeastMajorVersion8 = VERSION_PATTERN.matcher(dockerImageName.getVersionPart()).matches(); + if (isAtLeastMajorVersion8) { + // TLS using a self signed certificate is enabled by default in version 8 + // to prevent the HttpsUrlConnection to fail with certificate errors we use + // the log message wait strategy, as there will be a single JSON encoded message + // marking the node as started + setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*\"message\":\"started\".*")); + withPassword(ELASTICSEARCH_DEFAULT_PASSWORD); + } else { + setWaitStrategy(new HttpWaitStrategy() + .forPort(ELASTICSEARCH_DEFAULT_PORT) + .forStatusCodeMatching(response -> response == HTTP_OK || response == HTTP_UNAUTHORIZED) + .withStartupTimeout(Duration.ofMinutes(2))); + } + } + + @Override + protected void containerIsStarted(InspectContainerResponse containerInfo) { + if (isAtLeastMajorVersion8) { + byte[] bytes = copyFileFromContainer("/usr/share/elasticsearch/config/certs/http_ca.crt", IOUtils::toByteArray); + if (bytes.length > 0) { + this.caCertAsBytes = Optional.of(bytes); + } + } + } + + /** + * If this is running above Elasticsearch 8, the probably self signed CA cert will be extracted + * + * @return byte array optional containing the CA cert extracted from the docker container + */ + public Optional caCertAsBytes() { + return caCertAsBytes; + } + + /** + * A SSL context based on the self signed CA, so that using this SSL Context allows to connect to the Elasticsearch service + * @return a customized SSL Context + */ + public SSLContext createSslContextFromCa() { + try { + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + Certificate trustedCa = factory.generateCertificate(new ByteArrayInputStream(caCertAsBytes.get())); + KeyStore trustStore = KeyStore.getInstance("pkcs12"); + trustStore.load(null, null); + trustStore.setCertificateEntry("ca", trustedCa); + + final SSLContext sslContext = SSLContext.getInstance("TLSv1.3"); + TrustManagerFactory tmfactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmfactory.init(trustStore); + sslContext.init(null, tmfactory.getTrustManagers(), null); + return sslContext; + } catch (Exception e) { + throw new RuntimeException(e); + } } /** - * Define the Elasticsearch password to set. It enables security behind the scene. + * Define the Elasticsearch password to set. It enables security behind the scene for major version below 8.0.0. * It's not possible to use security with the oss image. * @param password Password to set * @return this @@ -92,7 +163,10 @@ public ElasticsearchContainer withPassword(String password) { "Please switch to the default distribution"); } withEnv("ELASTIC_PASSWORD", password); - withEnv("xpack.security.enabled", "true"); + if (!isAtLeastMajorVersion8) { + // major version 8 is secure by default and does not need this to enable authentication + withEnv("xpack.security.enabled", "true"); + } return this; } diff --git a/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java index 859587645d7..bbc1bb265c9 100644 --- a/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java +++ b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java @@ -1,11 +1,5 @@ package org.testcontainers.elasticsearch; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.rnorth.visibleassertions.VisibleAssertions.assertThrows; - -import java.io.IOException; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; @@ -25,6 +19,13 @@ import org.junit.Test; import org.testcontainers.utility.DockerImageName; +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.rnorth.visibleassertions.VisibleAssertions.assertThrows; + public class ElasticsearchContainerTest { /** @@ -249,6 +250,33 @@ public void incompatibleSettingsTest() { ); } + @Test + public void testElasticsearch8SecureByDefault() throws Exception { + try (ElasticsearchContainer container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:8.0.0")) { + // Start the container. This step might take some time... + container.start(); + + // Create the secured client. + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(ELASTICSEARCH_USERNAME, ElasticsearchContainer.ELASTICSEARCH_DEFAULT_PASSWORD)); + + client = RestClient.builder(HttpHost.create("https://" + container.getHttpHostAddress())) + .setHttpClientConfigCallback(httpClientBuilder -> { + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + httpClientBuilder.setSSLContext(container.createSslContextFromCa()); + return httpClientBuilder; + }) + .build(); + + Response response = client.performRequest(new Request("GET", "/_cluster/health")); + // }} + assertThat(response.getStatusLine().getStatusCode(), is(200)); + assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name")); + // httpClientSecuredContainer {{ + } + } + private RestClient getClient(ElasticsearchContainer container) { if (client == null) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); From 148559b71e4e5bb4d20c6d6df05c8d329e5570dc Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 21 Feb 2022 11:10:13 +0100 Subject: [PATCH 2/3] review comments --- .../elasticsearch/ElasticsearchContainer.java | 28 ++++++------------- .../ElasticsearchContainerTest.java | 2 -- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java index f95b904a597..a7c735c2d33 100644 --- a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java +++ b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java @@ -3,9 +3,9 @@ import com.github.dockerjava.api.command.InspectContainerResponse; import org.apache.commons.io.IOUtils; import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.Base58; +import org.testcontainers.utility.ComparableVersion; import org.testcontainers.utility.DockerImageName; import javax.net.ssl.SSLContext; @@ -15,12 +15,7 @@ import java.security.KeyStore; import java.security.cert.Certificate; import java.security.cert.CertificateFactory; -import java.time.Duration; import java.util.Optional; -import java.util.regex.Pattern; - -import static java.net.HttpURLConnection.HTTP_OK; -import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; /** * Represents an elasticsearch docker instance which exposes by default port 9200 and 9300 (transport.tcp.port) @@ -57,9 +52,6 @@ public class ElasticsearchContainer extends GenericContainer caCertAsBytes = Optional.empty(); @@ -94,19 +86,15 @@ public ElasticsearchContainer(final DockerImageName dockerImageName) { withNetworkAliases("elasticsearch-" + Base58.randomString(6)); withEnv("discovery.type", "single-node"); addExposedPorts(ELASTICSEARCH_DEFAULT_PORT, ELASTICSEARCH_DEFAULT_TCP_PORT); - this.isAtLeastMajorVersion8 = VERSION_PATTERN.matcher(dockerImageName.getVersionPart()).matches(); + this.isAtLeastMajorVersion8 = new ComparableVersion(dockerImageName.getVersionPart()).isGreaterThanOrEqualTo("8.0.0"); + // regex that + // matches 8.0 JSON logging with no whitespace between message field and content + // matches 7.x JSON logging with whitespace between message field and content + // matches 6.x text logging with node name in brackets and just a 'started' message till the end of the line + String regex = ".*(\"message\":\\s?\"started\".*|] started\n$)"; + setWaitStrategy(new LogMessageWaitStrategy().withRegEx(regex)); if (isAtLeastMajorVersion8) { - // TLS using a self signed certificate is enabled by default in version 8 - // to prevent the HttpsUrlConnection to fail with certificate errors we use - // the log message wait strategy, as there will be a single JSON encoded message - // marking the node as started - setWaitStrategy(new LogMessageWaitStrategy().withRegEx(".*\"message\":\"started\".*")); withPassword(ELASTICSEARCH_DEFAULT_PASSWORD); - } else { - setWaitStrategy(new HttpWaitStrategy() - .forPort(ELASTICSEARCH_DEFAULT_PORT) - .forStatusCodeMatching(response -> response == HTTP_OK || response == HTTP_UNAUTHORIZED) - .withStartupTimeout(Duration.ofMinutes(2))); } } diff --git a/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java index bbc1bb265c9..956c794135b 100644 --- a/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java +++ b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java @@ -270,10 +270,8 @@ public void testElasticsearch8SecureByDefault() throws Exception { .build(); Response response = client.performRequest(new Request("GET", "/_cluster/health")); - // }} assertThat(response.getStatusLine().getStatusCode(), is(200)); assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name")); - // httpClientSecuredContainer {{ } } From 5929083e28e45cc5b950380fb3745f3074ef63d8 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Wed, 6 Apr 2022 18:17:36 +0200 Subject: [PATCH 3/3] Update modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java Co-authored-by: Kevin Wittek --- .../testcontainers/elasticsearch/ElasticsearchContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java index a7c735c2d33..84881b2e87d 100644 --- a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java +++ b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java @@ -109,7 +109,7 @@ protected void containerIsStarted(InspectContainerResponse containerInfo) { } /** - * If this is running above Elasticsearch 8, the probably self signed CA cert will be extracted + * If this is running above Elasticsearch 8, this will return the probably self-signed CA cert that has been extracted * * @return byte array optional containing the CA cert extracted from the docker container */