From 414a5a081aadfe1659ec78edfafbf096680d0ee3 Mon Sep 17 00:00:00 2001 From: Richard North Date: Mon, 30 Apr 2018 09:16:51 +0100 Subject: [PATCH 01/16] Docker authentiation using credential store/helpers --- .gitignore | 2 + CHANGELOG.md | 1 + .../images/RemoteDockerImage.java | 6 + .../utility/DockerImageName.java | 10 +- .../utility/RegistryAuthLocator.java | 115 ++++++++++++++++++ .../dockerclient/ImagePullTest.java | 2 +- .../utility/RegistryAuthLocatorTest.java | 64 ++++++++++ .../config-with-helper-and-store.json | 12 ++ .../auth-config/config-with-helper.json | 11 ++ .../auth-config/config-with-store.json | 9 ++ .../auth-config/docker-credential-fake | 13 ++ 11 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java create mode 100644 core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java create mode 100644 core/src/test/resources/auth-config/config-with-helper-and-store.json create mode 100644 core/src/test/resources/auth-config/config-with-helper.json create mode 100644 core/src/test/resources/auth-config/config-with-store.json create mode 100755 core/src/test/resources/auth-config/docker-credential-fake diff --git a/.gitignore b/.gitignore index 4c3f24f5bb4..c95a6ece0cd 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,8 @@ node_modules/ .gradle/ build/ +out/ +*.class # Eclipse IDE files **/.project diff --git a/CHANGELOG.md b/CHANGELOG.md index f49c8b5b753..daa3ca539f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ All notable changes to this project will be documented in this file. ## [1.7.2] - 2018-04-30 ### Fixed +- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) - Retry any exceptions (not just `DockerClientException`) on image pull ([\#662](https://github.com/testcontainers/testcontainers-java/issues/662)) - Fixed handling of the paths with `+` in them ([\#664](https://github.com/testcontainers/testcontainers-java/issues/664)) diff --git a/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java b/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java index 9554793c004..bb0d0c969ce 100644 --- a/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java +++ b/core/src/main/java/org/testcontainers/images/RemoteDockerImage.java @@ -3,6 +3,7 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.ListImagesCmd; import com.github.dockerjava.api.exception.DockerClientException; +import com.github.dockerjava.api.model.AuthConfig; import com.github.dockerjava.api.model.Image; import com.github.dockerjava.core.command.PullImageResultCallback; import lombok.NonNull; @@ -14,6 +15,7 @@ import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; import org.testcontainers.utility.LazyFuture; +import org.testcontainers.utility.RegistryAuthLocator; import java.util.HashSet; import java.util.List; @@ -93,10 +95,14 @@ protected final String resolve() { // The image is not available locally - pull it try { + final RegistryAuthLocator authLocator = new RegistryAuthLocator(dockerClient.authConfig()); + final AuthConfig effectiveAuthConfig = authLocator.lookupAuthConfig(imageName); + final PullImageResultCallback callback = new PullImageResultCallback(); dockerClient .pullImageCmd(imageName.getUnversionedPart()) .withTag(imageName.getVersionPart()) + .withAuthConfig(effectiveAuthConfig) .exec(callback); callback.awaitCompletion(); AVAILABLE_IMAGE_NAME_CACHE.add(imageName); diff --git a/core/src/main/java/org/testcontainers/utility/DockerImageName.java b/core/src/main/java/org/testcontainers/utility/DockerImageName.java index 50b2d57b743..3f6d7124891 100644 --- a/core/src/main/java/org/testcontainers/utility/DockerImageName.java +++ b/core/src/main/java/org/testcontainers/utility/DockerImageName.java @@ -94,7 +94,11 @@ public String getVersionPart() { @Override public String toString() { - return getUnversionedPart() + versioning.getSeparator() + versioning.toString(); + if (versioning == null) { + return getUnversionedPart(); + } else { + return getUnversionedPart() + versioning.getSeparator() + versioning.toString(); + } } /** @@ -116,6 +120,10 @@ public void assertValid() { } } + public String getRegistry() { + return registry; + } + private interface Versioning { boolean isValid(); String getSeparator(); diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java new file mode 100644 index 00000000000..20646ae0c0a --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -0,0 +1,115 @@ +package org.testcontainers.utility; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.model.AuthConfig; +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.zeroturnaround.exec.ProcessExecutor; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.util.concurrent.TimeUnit; + +import static org.slf4j.LoggerFactory.getLogger; + +/** + * Utility to look up registry authentication information for an image. + */ +public class RegistryAuthLocator { + + private static final Logger log = getLogger(RegistryAuthLocator.class); + + private final AuthConfig defaultAuthConfig; + private final File configFile; + private final String commandPathPrefix; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @VisibleForTesting + RegistryAuthLocator(AuthConfig defaultAuthConfig, File configFile, String commandPathPrefix) { + this.defaultAuthConfig = defaultAuthConfig; + this.configFile = configFile; + this.commandPathPrefix = commandPathPrefix; + } + + /** + * @param defaultAuthConfig an AuthConfig object that should be returned if there is no overriding authentication + * available for images that are looked up + */ + public RegistryAuthLocator(AuthConfig defaultAuthConfig) { + this.defaultAuthConfig = defaultAuthConfig; + this.configFile = new File(System.getProperty("user.home") + "/.docker/config.json"); + this.commandPathPrefix = ""; + } + + /** + * Looks up an AuthConfig for a given image name. + * + * @param dockerImageName image name to be looked up (potentially including a registry URL part) + * @return an AuthConfig that is applicable to this specific image OR the defaultAuthConfig that has been set for + * this {@link RegistryAuthLocator}. + */ + public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { + log.debug("Looking up auth config for image: {}", dockerImageName); + try { + final JsonNode config = OBJECT_MAPPER.readTree(configFile); + + final String reposName = dockerImageName.getRegistry(); + final JsonNode auths = config.at("/auths/" + reposName); + + if (!auths.isMissingNode() && auths.size() == 0) { + // auths/ is an empty dict - use a credential helper + return authConfigUsingCredentialsStoreOrHelper(reposName, config); + } + } catch (Exception e) { + log.error("Failure when attempting to lookup auth config. Falling back to docker-java default behaviour", e); + } + return defaultAuthConfig; + } + + private AuthConfig authConfigUsingCredentialsStoreOrHelper(String hostName, JsonNode config) throws Exception { + + final String credsStoreName = config.at("/credsStore").asText(); + final String credHelper = config.at("/credHelpers/" + hostName).asText(); + + if (StringUtils.isNotBlank(credHelper)) { + return runCredentialProvider(hostName, credHelper); + } else if (StringUtils.isNotBlank(credsStoreName)) { + return runCredentialProvider(hostName, credsStoreName); + } else { + throw new UnsupportedOperationException(); + } + } + + private AuthConfig runCredentialProvider(String hostName, String credHelper) throws Exception { + final String credentialHelperName = commandPathPrefix + "docker-credential-" + credHelper; + String data; + + log.debug("Executing docker credential helper: {} to locate auth config for: {}", + credentialHelperName, hostName); + + try { + data = new ProcessExecutor() + .command(credentialHelperName, "get") + .redirectInput(new ByteArrayInputStream(hostName.getBytes())) + .readOutput(true) + .exitValueNormal() + .timeout(30, TimeUnit.SECONDS) + .execute() + .outputUTF8() + .trim(); + } catch (Exception e) { + log.error("Failure running docker credential helper ({})", credentialHelperName); + throw e; + } + + final JsonNode helperResponse = OBJECT_MAPPER.readTree(data); + log.debug("Credential helper provided auth config for: {}", hostName); + + return new AuthConfig() + .withRegistryAddress(helperResponse.at("/ServerURL").asText()) + .withUsername(helperResponse.at("/Username").asText()) + .withPassword(helperResponse.at("/Secret").asText()); + } +} diff --git a/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java b/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java index bfd03d2c9be..cb87caf16d2 100644 --- a/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java +++ b/core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java @@ -23,7 +23,7 @@ public static String[] parameters() { "gliderlabs/alpine@sha256:a19aa4a17a525c97e5a90a0c53a9f3329d2dc61b0a14df5447757a865671c085", "quay.io/testcontainers/ryuk:latest", "quay.io/testcontainers/ryuk:0.2.2", - "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da" + "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da", }; } diff --git a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java new file mode 100644 index 00000000000..f5f719c96ef --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java @@ -0,0 +1,64 @@ +package org.testcontainers.utility; + +import com.github.dockerjava.api.model.AuthConfig; +import com.google.common.io.Resources; +import org.apache.commons.lang.SystemUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.net.URISyntaxException; +import java.nio.file.Paths; + +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; +import static org.rnorth.visibleassertions.VisibleAssertions.assertNull; + +public class RegistryAuthLocatorTest { + + @BeforeClass + public static void nonWindowsTest() throws Exception { + Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); + } + + @Test + public void lookupAuthConfigWithoutCredentials() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("unauthenticated.registry.org/org/repo")); + + assertEquals("Default docker registry URL is set on auth config", "https://index.docker.io/v1/", authConfig.getRegistryAddress()); + assertNull("No username is set", authConfig.getUsername()); + assertNull("No password is set", authConfig.getPassword()); + } + + @Test + public void lookupAuthConfigUsingStore() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @Test + public void lookupAuthConfigUsingHelper() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-helper.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @NotNull + private RegistryAuthLocator createTestAuthLocator(String configName) throws URISyntaxException { + final File configFile = Paths.get(Resources.getResource("auth-config/" + configName).toURI()).toFile(); + return new RegistryAuthLocator(new AuthConfig(), configFile, configFile.getParentFile().getAbsolutePath() + "/"); + } + +} diff --git a/core/src/test/resources/auth-config/config-with-helper-and-store.json b/core/src/test/resources/auth-config/config-with-helper-and-store.json new file mode 100644 index 00000000000..cd623369d78 --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-helper-and-store.json @@ -0,0 +1,12 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credsStore": "fake", + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-with-helper.json b/core/src/test/resources/auth-config/config-with-helper.json new file mode 100644 index 00000000000..eaa670e4296 --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-helper.json @@ -0,0 +1,11 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-with-store.json b/core/src/test/resources/auth-config/config-with-store.json new file mode 100644 index 00000000000..d165997e52f --- /dev/null +++ b/core/src/test/resources/auth-config/config-with-store.json @@ -0,0 +1,9 @@ +{ + "auths": { + "registry.example.com": {} + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credsStore": "fake" +} diff --git a/core/src/test/resources/auth-config/docker-credential-fake b/core/src/test/resources/auth-config/docker-credential-fake new file mode 100755 index 00000000000..83b87227e30 --- /dev/null +++ b/core/src/test/resources/auth-config/docker-credential-fake @@ -0,0 +1,13 @@ +#!/bin/sh + +if [[ $1 != "get" ]]; then + exit 1 +fi + +read > /dev/null + +echo '{' \ + ' "ServerURL": "url",' \ + ' "Username": "username",' \ + ' "Secret": "secret"' \ + '}' From ada02b2b433c0eb922d73f924de3c5c6d6a301f0 Mon Sep 17 00:00:00 2001 From: Sergei Egorov Date: Thu, 19 Apr 2018 21:36:12 +0200 Subject: [PATCH 02/16] add #648 to CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index daa3ca539f9..c1f0c82d407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ All notable changes to this project will be documented in this file. ## [1.7.2] - 2018-04-30 +- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) + ### Fixed - Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) - Retry any exceptions (not just `DockerClientException`) on image pull ([\#662](https://github.com/testcontainers/testcontainers-java/issues/662)) From 60a55f9092e25401685716c726d0c136d90644da Mon Sep 17 00:00:00 2001 From: Richard North Date: Fri, 20 Apr 2018 08:47:35 +0100 Subject: [PATCH 03/16] Docker authentiation using credential store/helpers --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1f0c82d407..79d5934727a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ All notable changes to this project will be documented in this file. - Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640)) - Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([\#643](https://github.com/testcontainers/testcontainers-java/pull/643)) - Fixed Docker host IP detection within docker container (detect only if not explicitly set) ([\#648](https://github.com/testcontainers/testcontainers-java/pull/648)) +- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) ### Changed - Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630)) From a181b8b3fea926e75529e1aec087954566d0723c Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 24 Apr 2018 13:46:50 +0100 Subject: [PATCH 04/16] Update following initial review --- CHANGELOG.md | 2 +- .../utility/RegistryAuthLocator.java | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79d5934727a..9cb73ecbed5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,7 +60,7 @@ All notable changes to this project will be documented in this file. - Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640)) - Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([\#643](https://github.com/testcontainers/testcontainers-java/pull/643)) - Fixed Docker host IP detection within docker container (detect only if not explicitly set) ([\#648](https://github.com/testcontainers/testcontainers-java/pull/648)) -- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) +- Add support for private repositories using docker credential stores/helpers ([PR \#647](https://github.com/testcontainers/testcontainers-java/pull/647), fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567)) ### Changed - Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630)) diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java index 20646ae0c0a..b5b50b65f94 100644 --- a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.dockerjava.api.model.AuthConfig; import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.zeroturnaround.exec.ProcessExecutor; @@ -39,7 +38,9 @@ public class RegistryAuthLocator { */ public RegistryAuthLocator(AuthConfig defaultAuthConfig) { this.defaultAuthConfig = defaultAuthConfig; - this.configFile = new File(System.getProperty("user.home") + "/.docker/config.json"); + final String dockerConfigLocation = System.getenv().getOrDefault("DOCKER_CONFIG", + System.getProperty("user.home") + "/.docker"); + this.configFile = new File(dockerConfigLocation + "/config.json"); this.commandPathPrefix = ""; } @@ -62,6 +63,7 @@ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { // auths/ is an empty dict - use a credential helper return authConfigUsingCredentialsStoreOrHelper(reposName, config); } + // otherwise, defaultAuthConfig should already contain any credentials available } catch (Exception e) { log.error("Failure when attempting to lookup auth config. Falling back to docker-java default behaviour", e); } @@ -70,13 +72,13 @@ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { private AuthConfig authConfigUsingCredentialsStoreOrHelper(String hostName, JsonNode config) throws Exception { - final String credsStoreName = config.at("/credsStore").asText(); - final String credHelper = config.at("/credHelpers/" + hostName).asText(); + final JsonNode credsStoreName = config.at("/credsStore"); + final JsonNode credHelper = config.at("/credHelpers/" + hostName); - if (StringUtils.isNotBlank(credHelper)) { - return runCredentialProvider(hostName, credHelper); - } else if (StringUtils.isNotBlank(credsStoreName)) { - return runCredentialProvider(hostName, credsStoreName); + if (!credHelper.isMissingNode()) { + return runCredentialProvider(hostName, credHelper.asText()); + } else if (!credsStoreName.isMissingNode()) { + return runCredentialProvider(hostName, credsStoreName.asText()); } else { throw new UnsupportedOperationException(); } From 06ef5f6217198896d314ddbcb1d8cb2a6927528f Mon Sep 17 00:00:00 2001 From: Richard North Date: Mon, 30 Apr 2018 18:45:12 +0100 Subject: [PATCH 05/16] Change exception in case of failure --- .../java/org/testcontainers/utility/RegistryAuthLocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java index b5b50b65f94..84ad8b9868e 100644 --- a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -80,7 +80,7 @@ private AuthConfig authConfigUsingCredentialsStoreOrHelper(String hostName, Json } else if (!credsStoreName.isMissingNode()) { return runCredentialProvider(hostName, credsStoreName.asText()); } else { - throw new UnsupportedOperationException(); + throw new IllegalStateException("Unsupported Docker config auths settings!"); } } From 80d34a85dc30ed870ab982a1dca345567b8f9e64 Mon Sep 17 00:00:00 2001 From: Richard North Date: Sun, 13 May 2018 22:10:35 +0100 Subject: [PATCH 06/16] Simplify test resource resolution --- .../org/testcontainers/utility/RegistryAuthLocatorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java index f5f719c96ef..3b9790c002e 100644 --- a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java +++ b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java @@ -57,7 +57,7 @@ public void lookupAuthConfigUsingHelper() throws URISyntaxException { @NotNull private RegistryAuthLocator createTestAuthLocator(String configName) throws URISyntaxException { - final File configFile = Paths.get(Resources.getResource("auth-config/" + configName).toURI()).toFile(); + final File configFile = new File(Resources.getResource("auth-config/" + configName).toURI()); return new RegistryAuthLocator(new AuthConfig(), configFile, configFile.getParentFile().getAbsolutePath() + "/"); } From 87d9a3aa1a6bc6e880dd4838413e3fb65246e77d Mon Sep 17 00:00:00 2001 From: Richard North Date: Mon, 14 May 2018 09:18:07 +0100 Subject: [PATCH 07/16] Add extra logging --- .../testcontainers/utility/RegistryAuthLocator.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java index 84ad8b9868e..d211e2dd3b1 100644 --- a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -53,6 +53,12 @@ public RegistryAuthLocator(AuthConfig defaultAuthConfig) { */ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { log.debug("Looking up auth config for image: {}", dockerImageName); + + log.debug("RegistryAuthLocator has configFile: {} ({}) and commandPathPrefix: {}", + configFile, + configFile.exists() ? "exists" : "does not exist", + commandPathPrefix); + try { final JsonNode config = OBJECT_MAPPER.readTree(configFile); @@ -65,7 +71,11 @@ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { } // otherwise, defaultAuthConfig should already contain any credentials available } catch (Exception e) { - log.error("Failure when attempting to lookup auth config. Falling back to docker-java default behaviour", e); + log.error("Failure when attempting to lookup auth config (dockerImageName: {}, configFile: {}. " + + "Falling back to docker-java default behaviour", + dockerImageName, + configFile, + e); } return defaultAuthConfig; } From 682b558607b1baa386a2e179da2e4eb45a487f1b Mon Sep 17 00:00:00 2001 From: Richard North Date: Fri, 15 Jun 2018 15:41:11 +0100 Subject: [PATCH 08/16] Add store_artifacts for debugging (experimental) --- circle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/circle.yml b/circle.yml index b216656417f..9d9a7ff67c7 100644 --- a/circle.yml +++ b/circle.yml @@ -14,6 +14,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit okhttp: steps: - checkout From 09ca2f9ce0a80c726814a2a16d64dda80cad6cb9 Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 15 May 2018 14:06:52 +0100 Subject: [PATCH 09/16] Upgrade zt-exec --- core/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/build.gradle b/core/build.gradle index 15a0300738f..577c717965b 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -105,6 +105,8 @@ dependencies { shaded 'javax.ws.rs:javax.ws.rs-api:2.0.1' shaded 'org.zeroturnaround:zt-exec:1.8' + shaded 'org.rnorth:tcp-unix-socket-proxy:1.0.2' + shaded 'org.zeroturnaround:zt-exec:1.10' shaded 'commons-lang:commons-lang:2.6' shaded 'commons-io:commons-io:2.5' shaded 'commons-codec:commons-codec:1.11' From a6eaf0d3da8222a329d7283d887d5b7b6b367d97 Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 15 May 2018 16:17:58 +0100 Subject: [PATCH 10/16] Explicitly use bash for fake credential helper --- core/src/test/resources/auth-config/docker-credential-fake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/resources/auth-config/docker-credential-fake b/core/src/test/resources/auth-config/docker-credential-fake index 83b87227e30..0cfa0dfb72d 100755 --- a/core/src/test/resources/auth-config/docker-credential-fake +++ b/core/src/test/resources/auth-config/docker-credential-fake @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash if [[ $1 != "get" ]]; then exit 1 From 53acc95f13684fa1aada678fc3c75aad74b85ab1 Mon Sep 17 00:00:00 2001 From: Richard North Date: Tue, 15 May 2018 16:55:39 +0100 Subject: [PATCH 11/16] Always store test artifacts --- circle.yml | 6 ++++++ .../org/testcontainers/utility/RegistryAuthLocatorTest.java | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 9d9a7ff67c7..625c5026db0 100644 --- a/circle.yml +++ b/circle.yml @@ -47,6 +47,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit modules-jdbc-test: steps: - checkout @@ -63,6 +65,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit selenium: steps: - checkout @@ -76,6 +80,8 @@ jobs: when: always - store_test_results: path: ~/junit + - store_artifacts: + path: ~/junit workflows: version: 2 diff --git a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java index 3b9790c002e..ec5dfaa2e88 100644 --- a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java +++ b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java @@ -10,7 +10,6 @@ import java.io.File; import java.net.URISyntaxException; -import java.nio.file.Paths; import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; import static org.rnorth.visibleassertions.VisibleAssertions.assertNull; From 878ac17ba59a72f506753ae48256c9bf30094c68 Mon Sep 17 00:00:00 2001 From: Stanislav Baiduzhyi Date: Tue, 29 May 2018 14:55:51 +0200 Subject: [PATCH 12/16] Fixing the logical condition on when to check for cred helpers. --- .../java/org/testcontainers/utility/RegistryAuthLocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java index d211e2dd3b1..af5448ea361 100644 --- a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -65,7 +65,7 @@ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { final String reposName = dockerImageName.getRegistry(); final JsonNode auths = config.at("/auths/" + reposName); - if (!auths.isMissingNode() && auths.size() == 0) { + if (!auths.isMissingNode() || auths.size() == 0) { // auths/ is an empty dict - use a credential helper return authConfigUsingCredentialsStoreOrHelper(reposName, config); } From 42a4c565cf050e578509e87976d3adc71308f71d Mon Sep 17 00:00:00 2001 From: Stanislav Baiduzhyi Date: Mon, 4 Jun 2018 11:15:03 +0200 Subject: [PATCH 13/16] Post-merge fix. --- core/build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 577c717965b..3781c90002f 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -104,8 +104,6 @@ dependencies { shaded 'com.squareup.okhttp3:okhttp:3.10.0' shaded 'javax.ws.rs:javax.ws.rs-api:2.0.1' - shaded 'org.zeroturnaround:zt-exec:1.8' - shaded 'org.rnorth:tcp-unix-socket-proxy:1.0.2' shaded 'org.zeroturnaround:zt-exec:1.10' shaded 'commons-lang:commons-lang:2.6' shaded 'commons-io:commons-io:2.5' From f48753c7113115c44ff7b2edc71d4bd51b4d45d3 Mon Sep 17 00:00:00 2001 From: Stanislav Baiduzhyi Date: Tue, 5 Jun 2018 14:08:17 +0200 Subject: [PATCH 14/16] Fallback between 3 supported types of authentication. --- .../utility/RegistryAuthLocator.java | 78 +++++++++++++++---- .../utility/RegistryAuthLocatorTest.java | 25 +++++- .../config-empty-auth-with-helper.json | 10 +++ .../resources/auth-config/config-empty.json | 5 ++ .../config-existing-auth-with-helper.json | 14 ++++ 5 files changed, 116 insertions(+), 16 deletions(-) create mode 100644 core/src/test/resources/auth-config/config-empty-auth-with-helper.json create mode 100644 core/src/test/resources/auth-config/config-empty.json create mode 100644 core/src/test/resources/auth-config/config-existing-auth-with-helper.json diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java index af5448ea361..ef840f73f4a 100644 --- a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -9,6 +9,8 @@ import java.io.ByteArrayInputStream; import java.io.File; +import java.util.Iterator; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.slf4j.LoggerFactory.getLogger; @@ -45,7 +47,14 @@ public RegistryAuthLocator(AuthConfig defaultAuthConfig) { } /** - * Looks up an AuthConfig for a given image name. + * Looks up an AuthConfig for a given image name.
+ * Lookup is performed in following order:
+ *
    + *
  1. {@code auths} is checked for existing credentials for the specified registry.
    2. + *
  2. if no existing auth is found, {@code credHelpers} are checked for helper for the specified registry.
    3. + *
  3. if no suitable {@code credHelpers} found, {@code credsStore} is used.
    4. + *
  4. if no {@code credsStore} is found then the default configuration is returned.
    5. + *
* * @param dockerImageName image name to be looked up (potentially including a registry URL part) * @return an AuthConfig that is applicable to this specific image OR the defaultAuthConfig that has been set for @@ -61,13 +70,21 @@ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { try { final JsonNode config = OBJECT_MAPPER.readTree(configFile); - final String reposName = dockerImageName.getRegistry(); - final JsonNode auths = config.at("/auths/" + reposName); - if (!auths.isMissingNode() || auths.size() == 0) { - // auths/ is an empty dict - use a credential helper - return authConfigUsingCredentialsStoreOrHelper(reposName, config); + final AuthConfig existingAuthConfig = findExistingAuthConfig(config, reposName); + if (existingAuthConfig != null) { + return existingAuthConfig; + } + // auths is empty, using helper: + final AuthConfig helperAuthConfig = authConfigUsingHelper(config, reposName); + if (helperAuthConfig != null) { + return helperAuthConfig; + } + // no credsHelper to use, using credsStore: + final AuthConfig storeAuthConfig = authConfigUsingStore(config, reposName); + if (storeAuthConfig != null) { + return storeAuthConfig; } // otherwise, defaultAuthConfig should already contain any credentials available } catch (Exception e) { @@ -80,18 +97,49 @@ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { return defaultAuthConfig; } - private AuthConfig authConfigUsingCredentialsStoreOrHelper(String hostName, JsonNode config) throws Exception { + private AuthConfig findExistingAuthConfig(final JsonNode config, final String reposName) throws Exception { + final Map.Entry entry = findAuthNode(config, reposName); + if (entry != null && entry.getValue() != null && entry.getValue().size() > 0) { + return OBJECT_MAPPER + .treeToValue(entry.getValue(), AuthConfig.class) + .withRegistryAddress(entry.getKey()); + } + return null; + } + + private AuthConfig authConfigUsingHelper(final JsonNode config, final String reposName) throws Exception { + final JsonNode credHelpers = config.get("credHelpers"); + if (credHelpers != null && credHelpers.size() > 0) { + final JsonNode helperNode = credHelpers.get(reposName); + if (helperNode != null && helperNode.isTextual()) { + final String helper = helperNode.asText(); + return runCredentialProvider(reposName, helper); + } + } + return null; + } - final JsonNode credsStoreName = config.at("/credsStore"); - final JsonNode credHelper = config.at("/credHelpers/" + hostName); + private AuthConfig authConfigUsingStore(final JsonNode config, final String reposName) throws Exception { + final JsonNode credsStoreNode = config.get("credsStore"); + if (credsStoreNode != null && !credsStoreNode.isMissingNode() && credsStoreNode.isTextual()) { + final String credsStore = credsStoreNode.asText(); + return runCredentialProvider(reposName, credsStore); + } + return null; + } - if (!credHelper.isMissingNode()) { - return runCredentialProvider(hostName, credHelper.asText()); - } else if (!credsStoreName.isMissingNode()) { - return runCredentialProvider(hostName, credsStoreName.asText()); - } else { - throw new IllegalStateException("Unsupported Docker config auths settings!"); + private Map.Entry findAuthNode(final JsonNode config, final String reposName) throws Exception { + final JsonNode auths = config.get("auths"); + if (auths != null && auths.size() > 0) { + final Iterator> fields = auths.fields(); + while (fields.hasNext()) { + final Map.Entry entry = fields.next(); + if (entry.getKey().endsWith("://" + reposName)) { + return entry; + } + } } + return null; } private AuthConfig runCredentialProvider(String hostName, String credHelper) throws Exception { diff --git a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java index ec5dfaa2e88..7a79f6bd40c 100644 --- a/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java +++ b/core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java @@ -23,7 +23,7 @@ public static void nonWindowsTest() throws Exception { @Test public void lookupAuthConfigWithoutCredentials() throws URISyntaxException { - final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json"); + final RegistryAuthLocator authLocator = createTestAuthLocator("config-empty.json"); final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("unauthenticated.registry.org/org/repo")); @@ -54,6 +54,29 @@ public void lookupAuthConfigUsingHelper() throws URISyntaxException { assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); } + @Test + public void lookupUsingHelperEmptyAuth() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-empty-auth-with-helper.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress()); + assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername()); + assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword()); + } + + @Test + public void lookupNonEmptyAuthWithHelper() throws URISyntaxException { + final RegistryAuthLocator authLocator = createTestAuthLocator("config-existing-auth-with-helper.json"); + + final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo")); + + assertEquals("Correct server URL is obtained from a credential store", "https://registry.example.com", authConfig.getRegistryAddress()); + assertNull("No username is set", authConfig.getUsername()); + assertEquals("Correct email is obtained from a credential store", "not@val.id", authConfig.getEmail()); + assertEquals("Correct auth is obtained from a credential store", "encoded auth token", authConfig.getAuth()); + } + @NotNull private RegistryAuthLocator createTestAuthLocator(String configName) throws URISyntaxException { final File configFile = new File(Resources.getResource("auth-config/" + configName).toURI()); diff --git a/core/src/test/resources/auth-config/config-empty-auth-with-helper.json b/core/src/test/resources/auth-config/config-empty-auth-with-helper.json new file mode 100644 index 00000000000..8d8864815e3 --- /dev/null +++ b/core/src/test/resources/auth-config/config-empty-auth-with-helper.json @@ -0,0 +1,10 @@ +{ + "auths": { + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credHelpers": { + "registry.example.com": "fake" + } +} diff --git a/core/src/test/resources/auth-config/config-empty.json b/core/src/test/resources/auth-config/config-empty.json new file mode 100644 index 00000000000..38a103a2459 --- /dev/null +++ b/core/src/test/resources/auth-config/config-empty.json @@ -0,0 +1,5 @@ +{ + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + } +} diff --git a/core/src/test/resources/auth-config/config-existing-auth-with-helper.json b/core/src/test/resources/auth-config/config-existing-auth-with-helper.json new file mode 100644 index 00000000000..2c877b88f90 --- /dev/null +++ b/core/src/test/resources/auth-config/config-existing-auth-with-helper.json @@ -0,0 +1,14 @@ +{ + "auths": { + "https://registry.example.com": { + "email": "not@val.id", + "auth": "encoded auth token" + } + }, + "HttpHeaders": { + "User-Agent": "Docker-Client/18.03.0-ce (darwin)" + }, + "credHelpers": { + "registry.example.com": "fake" + } +} From 42fc4cfba79f24d732655ea5061c875a38c73de6 Mon Sep 17 00:00:00 2001 From: Stanislav Baiduzhyi Date: Tue, 5 Jun 2018 17:19:08 +0200 Subject: [PATCH 15/16] Fixing javadoc. --- .../java/org/testcontainers/utility/RegistryAuthLocator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java index ef840f73f4a..663633292ce 100644 --- a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -47,8 +47,9 @@ public RegistryAuthLocator(AuthConfig defaultAuthConfig) { } /** - * Looks up an AuthConfig for a given image name.
- * Lookup is performed in following order:
+ * Looks up an AuthConfig for a given image name. + *

+ * Lookup is performed in following order: *

    *
  1. {@code auths} is checked for existing credentials for the specified registry.
    2. *
  2. if no existing auth is found, {@code credHelpers} are checked for helper for the specified registry.
    3. From 7de6be82d2aead158a6f93b0bc90de8edac38abd Mon Sep 17 00:00:00 2001 From: Richard North Date: Fri, 15 Jun 2018 15:40:37 +0100 Subject: [PATCH 16/16] Disable untested feature on Windows - see https://github.com/testcontainers/testcontainers-java/issues/756 --- .../org/testcontainers/utility/RegistryAuthLocator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java index 663633292ce..eb290cae115 100644 --- a/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java +++ b/core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.dockerjava.api.model.AuthConfig; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.SystemUtils; import org.slf4j.Logger; import org.zeroturnaround.exec.ProcessExecutor; @@ -62,6 +63,13 @@ public RegistryAuthLocator(AuthConfig defaultAuthConfig) { * this {@link RegistryAuthLocator}. */ public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) { + + if (SystemUtils.IS_OS_WINDOWS) { + log.debug("RegistryAuthLocator is not supported on Windows. Please help test or improve it and update " + + "https://github.com/testcontainers/testcontainers-java/issues/756"); + return defaultAuthConfig; + } + log.debug("Looking up auth config for image: {}", dockerImageName); log.debug("RegistryAuthLocator has configFile: {} ({}) and commandPathPrefix: {}",