Pre-pull images required for Docker Compose image builds - including authenticated pulls

core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java

@@ -163,12 +163,13 @@ private void pullImages() {
         // (a) as a workaround for https://github.com/docker/compose/issues/5854, which prevents authenticated image pulls being possible when credential helpers are in use
         // (b) so that credential helper-based auth still works when compose is running from within a container
         parsedComposeFiles.stream()
-            .flatMap(it -> it.getServiceImageNames().stream())
+            .flatMap(it -> it.getDependencyImageNames().stream())
             .forEach(imageName -> {
                 try {
+                    log.info("Preemptively checking local images for '{}', referenced via a compose file or transitive Dockerfile. If not available, it will be pulled.", imageName);
                     DockerClientFactory.instance().checkAndPullImage(dockerClient, imageName);
                 } catch (Exception e) {
-                    log.warn("Failed to pull image '{}'. Exception message was {}", imageName, e.getMessage());
+                    log.warn("Unable to pre-fetch an image ({}) depended upon by Docker Compose build - startup will continue but may fail. Exception message was: {}", imageName, e.getMessage());
                 }
             });
     }

core/src/main/java/org/testcontainers/containers/ParsedDockerComposeFile.java

@@ -5,10 +5,13 @@
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
+import org.testcontainers.images.ParsedDockerfile;
 import org.yaml.snakeyaml.Yaml;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -23,9 +26,10 @@ class ParsedDockerComposeFile {
 
     private final Map<String, Object> composeFileContent;
     private final String composeFileName;
+    private final File composeFile;
 
     @Getter
-    private Set<String> serviceImageNames = new HashSet<>();
+    private Set<String> dependencyImageNames = new HashSet<>();
 
     ParsedDockerComposeFile(File composeFile) {
         Yaml yaml = new Yaml();
@@ -35,14 +39,15 @@ class ParsedDockerComposeFile {
             throw new IllegalArgumentException("Unable to parse YAML file from " + composeFile.getAbsolutePath(), e);
         }
         this.composeFileName = composeFile.getAbsolutePath();
-
+        this.composeFile = composeFile;
         parseAndValidate();
     }
 
     @VisibleForTesting
     ParsedDockerComposeFile(Map<String, Object> testContent) {
         this.composeFileContent = testContent;
         this.composeFileName = "";
+        this.composeFile = new File(".");
 
         parseAndValidate();
     }
@@ -80,15 +85,61 @@ private void parseAndValidate() {
             }
 
             final Map serviceDefinitionMap = (Map) serviceDefinition;
-            if (serviceDefinitionMap.containsKey("container_name")) {
-                throw new IllegalStateException(String.format(
-                    "Compose file %s has 'container_name' property set for service '%s' but this property is not supported by Testcontainers, consider removing it",
-                    composeFileName,
-                    serviceName
-                ));
+
+            validateNoContainerNameSpecified(serviceName, serviceDefinitionMap);
+            findServiceImageName(serviceDefinitionMap);
+            findImageNamesInDockerfile(serviceDefinitionMap);
+        }
+    }
+
+    private void validateNoContainerNameSpecified(String serviceName, Map serviceDefinitionMap) {
+        if (serviceDefinitionMap.containsKey("container_name")) {
+            throw new IllegalStateException(String.format(
+                "Compose file %s has 'container_name' property set for service '%s' but this property is not supported by Testcontainers, consider removing it",
+                composeFileName,
+                serviceName
+            ));
+        }
+    }
+
+    private void findServiceImageName(Map serviceDefinitionMap) {
+        if (serviceDefinitionMap.containsKey("image") && serviceDefinitionMap.get("image") instanceof String) {
+            final String imageName = (String) serviceDefinitionMap.get("image");
+            log.debug("Resolved dependency image for Docker Compose in {}: {}", composeFileName, imageName);
+            dependencyImageNames.add(imageName);
+        }
+    }
+
+    private void findImageNamesInDockerfile(Map serviceDefinitionMap) {
+        final Object buildNode = serviceDefinitionMap.get("build");
+        Path dockerfilePath = null;
+
+        if (buildNode instanceof Map) {
+            final Map buildElement = (Map) buildNode;
+            final Object dockerfileRelativePath = buildElement.get("dockerfile");
+            final Object contextRelativePath = buildElement.get("context");
+            if (dockerfileRelativePath instanceof String && contextRelativePath instanceof String) {
+                dockerfilePath = composeFile
+                    .getParentFile()
+                    .toPath()
+                    .resolve((String) contextRelativePath)
+                    .resolve((String) dockerfileRelativePath)
+                    .normalize();
             }
-            if (serviceDefinitionMap.containsKey("image") && serviceDefinitionMap.get("image") instanceof String) {
-                serviceImageNames.add((String) serviceDefinitionMap.get("image"));
+        } else if (buildNode instanceof String) {
+            dockerfilePath = composeFile
+                .getParentFile()
+                .toPath()
+                .resolve((String) buildNode)
+                .resolve("./Dockerfile")
+                .normalize();
+        }
+
+        if (dockerfilePath != null && Files.exists(dockerfilePath)) {
+            Set<String> resolvedImageNames = new ParsedDockerfile(dockerfilePath).getDependencyImageNames();
+            if (!resolvedImageNames.isEmpty()) {
+                log.debug("Resolved Dockerfile dependency images for Docker Compose in {} -> {}: {}", composeFileName, dockerfilePath, resolvedImageNames);
+                this.dependencyImageNames.addAll(resolvedImageNames);
             }
         }
     }

core/src/main/java/org/testcontainers/images/ParsedDockerfile.java

@@ -0,0 +1,67 @@
+package org.testcontainers.images;
+
+import com.google.common.annotations.VisibleForTesting;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Representation of a Dockerfile, with partial parsing for extraction of a minimal set of data.
+ */
+@Slf4j
+public class ParsedDockerfile {
+
+    private static final Pattern FROM_LINE_PATTERN = Pattern.compile("FROM ([^\\s]+).*");
+
+    private final Path dockerFilePath;
+
+    @Getter
+    private Set<String> dependencyImageNames = Collections.emptySet();
+
+    public ParsedDockerfile(Path dockerFilePath) {
+        this.dockerFilePath = dockerFilePath;
+        parse(read());
+    }
+
+    @VisibleForTesting
+    ParsedDockerfile(List<String> lines) {
+        this.dockerFilePath = Paths.get("dummy.Dockerfile");
+        parse(lines);
+    }
+
+    private List<String> read() {
+        if (!Files.exists(dockerFilePath)) {
+            log.warn("Tried to parse Dockerfile at path {} but none was found", dockerFilePath);
+            return Collections.emptyList();
+        }
+
+        try {
+            return Files.readAllLines(dockerFilePath);
+        } catch (IOException e) {
+            log.warn("Unable to read Dockerfile at path {}", dockerFilePath, e);
+            return Collections.emptyList();
+        }
+    }
+
+    private void parse(List<String> lines) {
+        dependencyImageNames = lines.stream()
+            .map(FROM_LINE_PATTERN::matcher)
+            .filter(Matcher::matches)
+            .map(matcher -> matcher.group(1))
+            .collect(Collectors.toSet());
+
+        if (!dependencyImageNames.isEmpty()) {
+            log.debug("Found dependency images in Dockerfile {}: {}", dockerFilePath, dependencyImageNames);
+        }
+    }
+}

core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java

@@ -2,10 +2,8 @@
 
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.BuildImageCmd;
-import com.github.dockerjava.api.exception.DockerClientException;
 import com.github.dockerjava.api.model.BuildResponseItem;
 import com.github.dockerjava.core.command.BuildImageResultCallback;
-import com.google.common.collect.Sets;
 import lombok.Cleanup;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -14,6 +12,7 @@
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.testcontainers.DockerClientFactory;
+import org.testcontainers.images.ParsedDockerfile;
 import org.testcontainers.images.builder.traits.BuildContextBuilderTrait;
 import org.testcontainers.images.builder.traits.ClasspathTrait;
 import org.testcontainers.images.builder.traits.DockerfileTrait;
@@ -28,6 +27,7 @@
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -51,6 +51,7 @@ public class ImageFromDockerfile extends LazyFuture<String> implements
     private final Map<String, String> buildArgs = new HashMap<>();
     private Optional<String> dockerFilePath = Optional.empty();
     private Optional<Path> dockerfile = Optional.empty();
+    private Set<String> dependencyImageNames = Collections.emptySet();
 
     public ImageFromDockerfile() {
         this("testcontainers/" + Base58.randomString(16).toLowerCase());
@@ -81,6 +82,17 @@ protected final String resolve() {
         Logger logger = DockerLoggerFactory.getLogger(dockerImageName);
 
         DockerClient dockerClient = DockerClientFactory.instance().client();
+
+        dependencyImageNames.forEach(imageName -> {
+            try {
+                log.info("Pre-emptively checking local images for '{}', referenced via a Dockerfile. If not available, it will be pulled.", imageName);
+                DockerClientFactory.instance().checkAndPullImage(dockerClient, imageName);
+            } catch (Exception e) {
+                log.warn("Unable to pre-fetch an image ({}) depended upon by Dockerfile - image build will continue but may fail. Exception message was: {}", imageName, e.getMessage());
+            }
+        });
+
+
         try {
             if (deleteOnExit) {
                 ResourceReaper.instance().registerImageForCleanup(dockerImageName);
@@ -139,7 +151,11 @@ public void onNext(BuildResponseItem item) {
     protected void configure(BuildImageCmd buildImageCmd) {
         buildImageCmd.withTag(this.getDockerImageName());
         this.dockerFilePath.ifPresent(buildImageCmd::withDockerfilePath);
-        this.dockerfile.ifPresent(p -> buildImageCmd.withDockerfile(p.toFile()));
+        this.dockerfile.ifPresent(p -> {
+            buildImageCmd.withDockerfile(p.toFile());
+            dependencyImageNames = new ParsedDockerfile(p).getDependencyImageNames();
+        });
+
         this.buildArgs.forEach(buildImageCmd::withBuildArg);
     }
 

core/src/test/java/org/testcontainers/containers/ParsedDockerComposeFileValidationTest.java

@@ -76,13 +76,27 @@ public void shouldIgnoreUnknownStructure() {
     public void shouldObtainImageNamesV1() {
         File file = new File("src/test/resources/docker-compose-imagename-parsing-v1.yml");
         ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
-        assertEquals("all defined service names are found", Sets.newHashSet("redis", "mysql"), parsedFile.getServiceImageNames());
+        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "postgres"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, postgres from Dockerfile build
     }
 
     @Test
     public void shouldObtainImageNamesV2() {
         File file = new File("src/test/resources/docker-compose-imagename-parsing-v2.yml");
         ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
-        assertEquals("all defined service names are found", Sets.newHashSet("redis", "mysql"), parsedFile.getServiceImageNames());
+        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "postgres"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, postgres from Dockerfile build
+    }
+
+    @Test
+    public void shouldObtainImageFromDockerfileBuild() {
+        File file = new File("src/test/resources/docker-compose-imagename-parsing-dockerfile.yml");
+        ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
+        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "alpine:3.2"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, alpine:3.2 from Dockerfile build
+    }
+
+    @Test
+    public void shouldObtainImageFromDockerfileBuildWithContext() {
+        File file = new File("src/test/resources/docker-compose-imagename-parsing-dockerfile-with-context.yml");
+        ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
+        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "alpine:3.2"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, alpine:3.2 from Dockerfile build
     }
 }

core/src/test/java/org/testcontainers/images/ParsedDockerfileTest.java

@@ -0,0 +1,54 @@
+package org.testcontainers.images;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import java.nio.file.Paths;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+
+public class ParsedDockerfileTest {
+
+    @Test
+    public void doesSimpleParsing() {
+        final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage", "RUN something"));
+        assertEquals("extracts a single image name", Sets.newHashSet("someimage"), parsedDockerfile.getDependencyImageNames());
+    }
+
+    @Test
+    public void handlesTags() {
+        final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage:tag", "RUN something"));
+        assertEquals("retains tags in image names", Sets.newHashSet("someimage:tag"), parsedDockerfile.getDependencyImageNames());
+    }
+
+    @Test
+    public void handlesDigests() {
+        final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage@sha256:abc123", "RUN something"));
+        assertEquals("retains digests in image names", Sets.newHashSet("someimage@sha256:abc123"), parsedDockerfile.getDependencyImageNames());
+    }
+
+    @Test
+    public void ignoringCommentedFromLines() {
+        final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage", "#FROM somethingelse"));
+        assertEquals("ignores commented from lines", Sets.newHashSet("someimage"), parsedDockerfile.getDependencyImageNames());
+    }
+
+    @Test
+    public void ignoringBuildStageNames() {
+        final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage --as=base", "RUN something", "FROM nextimage", "RUN something"));
+        assertEquals("ignores build stage names and allows multiple images to be extracted", Sets.newHashSet("someimage", "nextimage"), parsedDockerfile.getDependencyImageNames());
+    }
+
+    @Test
+    public void handlesGracefullyIfNoFromLine() {
+        final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("RUN something", "# is this even a valid Dockerfile?"));
+        assertEquals("handles invalid Dockerfiles gracefully", Sets.newHashSet(), parsedDockerfile.getDependencyImageNames());
+    }
+
+    @Test
+    public void handlesGracefullyIfDockerfileNotFound() {
+        final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(Paths.get("nonexistent.Dockerfile"));
+        assertEquals("handles missing Dockerfiles gracefully", Sets.newHashSet(), parsedDockerfile.getDependencyImageNames());
+    }
+}

core/src/test/resources/Dockerfile

@@ -0,0 +1 @@
+FROM postgres

core/src/test/resources/docker-compose-imagename-parsing-dockerfile-with-context.yml

@@ -0,0 +1,12 @@
+version: "2.1"
+services:
+    redis:
+        image: redis
+    mysql:
+        image: mysql
+    custom:
+        build:
+            context: compose-dockerfile
+            dockerfile: Dockerfile
+networks:
+    custom_network: {}

core/src/test/resources/docker-compose-imagename-parsing-dockerfile.yml

@@ -0,0 +1,10 @@
+version: "2.1"
+services:
+    redis:
+        image: redis
+    mysql:
+        image: mysql
+    custom:
+        build: compose-dockerfile
+networks:
+    custom_network: {}