Credential helpers

.gitignore

@@ -46,6 +46,8 @@ node_modules/
 
 .gradle/
 build/
+out/
+*.class
 
 # Eclipse IDE files
 **/.project

CHANGELOG.md

@@ -42,7 +42,10 @@ All notable changes to this project will be documented in this file.
 
 ## [1.7.2] - 2018-04-30
 
+- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))
+
 ### Fixed
+- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))
 - Retry any exceptions (not just `DockerClientException`) on image pull ([\#662](https://github.com/testcontainers/testcontainers-java/issues/662))
 - Fixed handling of the paths with `+` in them ([\#664](https://github.com/testcontainers/testcontainers-java/issues/664))
 
@@ -57,6 +60,7 @@ All notable changes to this project will be documented in this file.
 - Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640))
 - Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([\#643](https://github.com/testcontainers/testcontainers-java/pull/643))
 - Fixed Docker host IP detection within docker container (detect only if not explicitly set) ([\#648](https://github.com/testcontainers/testcontainers-java/pull/648))
+- Add support for private repositories using docker credential stores/helpers ([PR \#647](https://github.com/testcontainers/testcontainers-java/pull/647), fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))
 
 ### Changed
 - Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630))

circle.yml

@@ -14,6 +14,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
   okhttp:
     steps:
       - checkout
@@ -45,6 +47,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
   modules-jdbc-test:
     steps:
       - checkout
@@ -61,6 +65,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
   selenium:
     steps:
       - checkout
@@ -74,6 +80,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
 
 workflows:
   version: 2

core/build.gradle

@@ -104,7 +104,7 @@ dependencies {
     shaded 'com.squareup.okhttp3:okhttp:3.10.0'
 
     shaded 'javax.ws.rs:javax.ws.rs-api:2.0.1'
-    shaded 'org.zeroturnaround:zt-exec:1.8'
+    shaded 'org.zeroturnaround:zt-exec:1.10'
     shaded 'commons-lang:commons-lang:2.6'
     shaded 'commons-io:commons-io:2.5'
     shaded 'commons-codec:commons-codec:1.11'

core/src/main/java/org/testcontainers/images/RemoteDockerImage.java

@@ -3,6 +3,7 @@
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.ListImagesCmd;
 import com.github.dockerjava.api.exception.DockerClientException;
+import com.github.dockerjava.api.model.AuthConfig;
 import com.github.dockerjava.api.model.Image;
 import com.github.dockerjava.core.command.PullImageResultCallback;
 import lombok.NonNull;
@@ -14,6 +15,7 @@
 import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.DockerLoggerFactory;
 import org.testcontainers.utility.LazyFuture;
+import org.testcontainers.utility.RegistryAuthLocator;
 
 import java.util.HashSet;
 import java.util.List;
@@ -93,10 +95,14 @@ protected final String resolve() {
 
                 // The image is not available locally - pull it
                 try {
+                    final RegistryAuthLocator authLocator = new RegistryAuthLocator(dockerClient.authConfig());
+                    final AuthConfig effectiveAuthConfig = authLocator.lookupAuthConfig(imageName);
+
                     final PullImageResultCallback callback = new PullImageResultCallback();
                     dockerClient
                         .pullImageCmd(imageName.getUnversionedPart())
                         .withTag(imageName.getVersionPart())
+                        .withAuthConfig(effectiveAuthConfig)
                         .exec(callback);
                     callback.awaitCompletion();
                     AVAILABLE_IMAGE_NAME_CACHE.add(imageName);

core/src/main/java/org/testcontainers/utility/DockerImageName.java

@@ -94,7 +94,11 @@ public String getVersionPart() {
 
     @Override
     public String toString() {
-        return getUnversionedPart() + versioning.getSeparator() + versioning.toString();
+        if (versioning == null) {
+            return getUnversionedPart();
+        } else {
+            return getUnversionedPart() + versioning.getSeparator() + versioning.toString();
+        }
     }
 
     /**
@@ -116,6 +120,10 @@ public void assertValid() {
         }
     }
 
+    public String getRegistry() {
+        return registry;
+    }
+
     private interface Versioning {
         boolean isValid();
         String getSeparator();

core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java

@@ -0,0 +1,184 @@
+package org.testcontainers.utility;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.dockerjava.api.model.AuthConfig;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.SystemUtils;
+import org.slf4j.Logger;
+import org.zeroturnaround.exec.ProcessExecutor;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Utility to look up registry authentication information for an image.
+ */
+public class RegistryAuthLocator {
+
+    private static final Logger log = getLogger(RegistryAuthLocator.class);
+
+    private final AuthConfig defaultAuthConfig;
+    private final File configFile;
+    private final String commandPathPrefix;
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @VisibleForTesting
+    RegistryAuthLocator(AuthConfig defaultAuthConfig, File configFile, String commandPathPrefix) {
+        this.defaultAuthConfig = defaultAuthConfig;
+        this.configFile = configFile;
+        this.commandPathPrefix = commandPathPrefix;
+    }
+
+    /**
+     * @param defaultAuthConfig an AuthConfig object that should be returned if there is no overriding authentication
+     *                          available for images that are looked up
+     */
+    public RegistryAuthLocator(AuthConfig defaultAuthConfig) {
+        this.defaultAuthConfig = defaultAuthConfig;
+        final String dockerConfigLocation = System.getenv().getOrDefault("DOCKER_CONFIG",
+            System.getProperty("user.home") + "/.docker");
+        this.configFile = new File(dockerConfigLocation + "/config.json");
+        this.commandPathPrefix = "";
+    }
+
+    /**
+     * Looks up an AuthConfig for a given image name.
+     * <p>
+     * Lookup is performed in following order:
+     * <ol>
+     *     <li>{@code auths} is checked for existing credentials for the specified registry.</li>
+     *     <li>if no existing auth is found, {@code credHelpers} are checked for helper for the specified registry.</li>
+     *     <li>if no suitable {@code credHelpers} found, {@code credsStore} is used.</li>
+     *     <li>if no {@code credsStore} is found then the default configuration is returned.</li>
+     * </ol>
+     *
+     * @param dockerImageName image name to be looked up (potentially including a registry URL part)
+     * @return an AuthConfig that is applicable to this specific image OR the defaultAuthConfig that has been set for
+     * this {@link RegistryAuthLocator}.
+     */
+    public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) {
+
+        if (SystemUtils.IS_OS_WINDOWS) {
+            log.debug("RegistryAuthLocator is not supported on Windows. Please help test or improve it and update " +
+                "https://github.com/testcontainers/testcontainers-java/issues/756");
+            return defaultAuthConfig;
+        }
+
+        log.debug("Looking up auth config for image: {}", dockerImageName);
+
+        log.debug("RegistryAuthLocator has configFile: {} ({}) and commandPathPrefix: {}",
+            configFile,
+            configFile.exists() ? "exists" : "does not exist",
+            commandPathPrefix);
+
+        try {
+            final JsonNode config = OBJECT_MAPPER.readTree(configFile);
+            final String reposName = dockerImageName.getRegistry();
+
+            final AuthConfig existingAuthConfig = findExistingAuthConfig(config, reposName);
+            if (existingAuthConfig != null) {
+                return existingAuthConfig;
+            }
+            // auths is empty, using helper:
+            final AuthConfig helperAuthConfig = authConfigUsingHelper(config, reposName);
+            if (helperAuthConfig != null) {
+                return helperAuthConfig;
+            }
+            // no credsHelper to use, using credsStore:
+            final AuthConfig storeAuthConfig = authConfigUsingStore(config, reposName);
+            if (storeAuthConfig != null) {
+                return storeAuthConfig;
+            }
+            // otherwise, defaultAuthConfig should already contain any credentials available
+        } catch (Exception e) {
+            log.error("Failure when attempting to lookup auth config (dockerImageName: {}, configFile: {}. " +
+                "Falling back to docker-java default behaviour",
+                dockerImageName,
+                configFile,
+                e);
+        }
+        return defaultAuthConfig;
+    }
+
+    private AuthConfig findExistingAuthConfig(final JsonNode config, final String reposName) throws Exception {
+        final Map.Entry<String, JsonNode> entry = findAuthNode(config, reposName);
+        if (entry != null && entry.getValue() != null && entry.getValue().size() > 0) {
+            return OBJECT_MAPPER
+                .treeToValue(entry.getValue(), AuthConfig.class)
+                .withRegistryAddress(entry.getKey());
+        }
+        return null;
+    }
+
+    private AuthConfig authConfigUsingHelper(final JsonNode config, final String reposName) throws Exception {
+        final JsonNode credHelpers = config.get("credHelpers");
+        if (credHelpers != null && credHelpers.size() > 0) {
+            final JsonNode helperNode = credHelpers.get(reposName);
+            if (helperNode != null && helperNode.isTextual()) {
+                final String helper = helperNode.asText();
+                return runCredentialProvider(reposName, helper);
+            }
+        }
+        return null;
+    }
+
+    private AuthConfig authConfigUsingStore(final JsonNode config, final String reposName) throws Exception {
+        final JsonNode credsStoreNode = config.get("credsStore");
+        if (credsStoreNode != null && !credsStoreNode.isMissingNode() && credsStoreNode.isTextual()) {
+            final String credsStore = credsStoreNode.asText();
+            return runCredentialProvider(reposName, credsStore);
+        }
+        return null;
+    }
+
+    private Map.Entry<String, JsonNode> findAuthNode(final JsonNode config, final String reposName) throws Exception {
+        final JsonNode auths = config.get("auths");
+        if (auths != null && auths.size() > 0) {
+            final Iterator<Map.Entry<String, JsonNode>> fields = auths.fields();
+            while (fields.hasNext()) {
+                final Map.Entry<String, JsonNode> entry = fields.next();
+                if (entry.getKey().endsWith("://" + reposName)) {
+                    return entry;
+                }
+            }
+        }
+        return null;
+    }
+
+    private AuthConfig runCredentialProvider(String hostName, String credHelper) throws Exception {
+        final String credentialHelperName = commandPathPrefix + "docker-credential-" + credHelper;
+        String data;
+
+        log.debug("Executing docker credential helper: {} to locate auth config for: {}",
+            credentialHelperName, hostName);
+
+        try {
+            data = new ProcessExecutor()
+                .command(credentialHelperName, "get")
+                .redirectInput(new ByteArrayInputStream(hostName.getBytes()))
+                .readOutput(true)
+                .exitValueNormal()
+                .timeout(30, TimeUnit.SECONDS)
+                .execute()
+                .outputUTF8()
+                .trim();
+        } catch (Exception e) {
+            log.error("Failure running docker credential helper ({})", credentialHelperName);
+            throw e;
+        }
+
+        final JsonNode helperResponse = OBJECT_MAPPER.readTree(data);
+        log.debug("Credential helper provided auth config for: {}", hostName);
+
+        return new AuthConfig()
+            .withRegistryAddress(helperResponse.at("/ServerURL").asText())
+            .withUsername(helperResponse.at("/Username").asText())
+            .withPassword(helperResponse.at("/Secret").asText());
+    }
+}

core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java

@@ -23,7 +23,7 @@ public static String[] parameters() {
             "gliderlabs/alpine@sha256:a19aa4a17a525c97e5a90a0c53a9f3329d2dc61b0a14df5447757a865671c085",
             "quay.io/testcontainers/ryuk:latest",
             "quay.io/testcontainers/ryuk:0.2.2",
-            "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da"
+            "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da",
         };
     }