Skip to content

Commit

Permalink
Add Image substitution mechanism
Browse files Browse the repository at this point in the history
Builds upon #3021 and #3411:

* adds a pluggable image substitution mechanism using ServiceLoader, enabling users to perform custom substitution/auditing of images being used by their tests

* provides a default implementation that behaves similarly to legacy `TestcontainersConfiguration` approach (`testcontainers.properties`)

Notes:

* behaviour is similar but not quite identical to `TestcontainersConfiguration`: use of a configured custom image for, e.g. Kafka/Pulsar that does not have a tag specified causes the substitution to take effect for all usages. It seems very unlikely that people would be using a mix of the config file image overrides in some places _and_ specific images specified in code in others.

* Duplication of default image names in modules vs `TestcontainersConfiguration` class is intentional: specifying image overrides in `testcontainers.properties` should be removed in the future.

* ~Add log deprecation warnings when `testcontainers.properties` image overrides are used.~ Defer to a future release?
  • Loading branch information
rnorth committed Oct 29, 2020
1 parent 0f51dfa commit 950af34
Show file tree
Hide file tree
Showing 34 changed files with 575 additions and 113 deletions.
11 changes: 8 additions & 3 deletions core/src/main/java/org/testcontainers/DockerClientFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.testcontainers.dockerclient.TransportConfig;
import org.testcontainers.images.TimeLimitedLoggedPullImageResultCallback;
import org.testcontainers.utility.ComparableVersion;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.ImageNameSubstitutor;
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.ResourceReaper;
import org.testcontainers.utility.TestcontainersConfiguration;
Expand Down Expand Up @@ -61,7 +63,7 @@ public class DockerClientFactory {
TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID
);

private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString();
private static final DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5");
private static DockerClientFactory instance;

// Cached client configuration
Expand Down Expand Up @@ -343,8 +345,11 @@ public <T> T runInsideDocker(Consumer<CreateContainerCmd> createContainerCmdCons
}

private <T> T runInsideDocker(DockerClient client, Consumer<CreateContainerCmd> createContainerCmdConsumer, BiFunction<DockerClient, String, T> block) {
checkAndPullImage(client, TINY_IMAGE);
CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE)

final String tinyImage = ImageNameSubstitutor.instance().apply(TINY_IMAGE).asCanonicalNameString();

checkAndPullImage(client, tinyImage);
CreateContainerCmd createContainerCmd = client.createContainerCmd(tinyImage)
.withLabels(DEFAULT_LABELS);
createContainerCmdConsumer.accept(createContainerCmd);
String id = createContainerCmd.exec().getId();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
import org.testcontainers.utility.AuditLogger;
import org.testcontainers.utility.Base58;
import org.testcontainers.utility.CommandLine;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.LogUtils;
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.ResourceReaper;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.zeroturnaround.exec.InvalidExitValueException;
import org.zeroturnaround.exec.ProcessExecutor;
import org.zeroturnaround.exec.stream.slf4j.Slf4jStream;
Expand Down Expand Up @@ -608,10 +608,11 @@ interface DockerCompose {
class ContainerisedDockerCompose extends GenericContainer<ContainerisedDockerCompose> implements DockerCompose {

public static final char UNIX_PATH_SEPERATOR = ':';
public static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("docker/compose:1.24.1");

public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {

super(TestcontainersConfiguration.getInstance().getDockerComposeDockerImageName());
super(DEFAULT_IMAGE_NAME);
addEnv(ENV_PROJECT_NAME, identifier);

// Map the docker compose file into the container
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,13 +239,9 @@ public GenericContainer(@NonNull final RemoteDockerImage image) {
*/
@Deprecated
public GenericContainer() {
this(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString());
this(TestcontainersConfiguration.getInstance().getTinyImage());
}

/**
* @deprecated use {@link GenericContainer(DockerImageName)} instead
*/
@Deprecated
public GenericContainer(@NonNull final String dockerImageName) {
this.setDockerImageName(dockerImageName);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
import lombok.AccessLevel;
import lombok.Getter;
import lombok.SneakyThrows;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.testcontainers.utility.DockerImageName;

import java.time.Duration;
import java.util.AbstractMap;
import java.util.Collections;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

public enum PortForwardingContainer {
Expand All @@ -29,7 +29,7 @@ public enum PortForwardingContainer {
@SneakyThrows
private Connection createSSHSession() {
String password = UUID.randomUUID().toString();
container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdDockerImageName())
container = new GenericContainer<>(DockerImageName.parse("testcontainers/sshd:1.0.0"))
.withExposedPorts(22)
.withEnv("PASSWORD", password)
.withCommand(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package org.testcontainers.containers;

import org.testcontainers.utility.Base58;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.TestcontainersConfiguration;

import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import org.testcontainers.utility.Base58;
import org.testcontainers.utility.DockerImageName;

/**
* A socat container is used as a TCP proxy, enabling any TCP port of another container to be exposed
Expand All @@ -17,7 +15,7 @@ public class SocatContainer extends GenericContainer<SocatContainer> {
private final Map<Integer, String> targets = new HashMap<>();

public SocatContainer() {
this(TestcontainersConfiguration.getInstance().getSocatDockerImageName());
this(DockerImageName.parse("alpine/socat:1.7.3.4-r0"));
}

public SocatContainer(final DockerImageName dockerImageName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import lombok.ToString;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.testcontainers.utility.DockerImageName;

import java.io.File;
import java.io.InputStream;
Expand Down Expand Up @@ -52,7 +52,7 @@ public VncRecordingContainer(@NonNull GenericContainer<?> targetContainer) {
* Create a sidekick container and attach it to another container. The VNC output of that container will be recorded.
*/
public VncRecordingContainer(@NonNull Network network, @NonNull String targetNetworkAlias) throws IllegalStateException {
super(TestcontainersConfiguration.getInstance().getVncDockerImageName());
super(DockerImageName.parse("testcontainers/vnc-recorder:1.1.0"));

this.targetNetworkAlias = targetNetworkAlias;
withNetwork(network);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public static DockerClientProviderStrategy getFirstValidStrategy(List<DockerClie
}

if (strategy.isPersistable()) {
TestcontainersConfiguration.getInstance().updateGlobalConfig("docker.client.strategy", strategy.getClass().getName());
TestcontainersConfiguration.getInstance().updateUserConfig("docker.client.strategy", strategy.getClass().getName());
}

return Stream.of(strategy);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.testcontainers.containers.ContainerFetchException;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.ImageNameSubstitutor;
import org.testcontainers.utility.LazyFuture;

import java.time.Duration;
Expand Down Expand Up @@ -44,12 +45,12 @@ public RemoteDockerImage(DockerImageName dockerImageName) {

@Deprecated
public RemoteDockerImage(String dockerImageName) {
this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(dockerImageName));
this(DockerImageName.parse(dockerImageName));
}

@Deprecated
public RemoteDockerImage(@NonNull String repository, @NonNull String tag) {
this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(repository).withTag(tag));
this(DockerImageName.parse(repository).withTag(tag));
}

public RemoteDockerImage(@NonNull Future<String> imageFuture) {
Expand Down Expand Up @@ -100,7 +101,10 @@ protected final String resolve() {
}

private DockerImageName getImageName() throws InterruptedException, ExecutionException {
return imageNameFuture.get();
final DockerImageName specifiedImageName = imageNameFuture.get();

// Allow the image name to be substituted
return ImageNameSubstitutor.instance().apply(specifiedImageName);
}

@ToString.Include(name = "imageName", rank = 1)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

/**
* {@link ImageNameSubstitutor} which takes replacement image names from configuration.
* See {@link TestcontainersConfiguration} for the subset of image names which can be substituted using this mechanism.
* <p>
* WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader}
* requires it to be so. Public visibility DOES NOT make it part of the public API.
*/
@Slf4j
public class ConfigurationFileImageNameSubstitutor extends ImageNameSubstitutor {

private final TestcontainersConfiguration configuration;

public ConfigurationFileImageNameSubstitutor() {
this(TestcontainersConfiguration.getInstance());
}

@VisibleForTesting
ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration configuration) {
this.configuration = configuration;
}

@Override
public DockerImageName apply(final DockerImageName original) {
final DockerImageName result = configuration
.getConfiguredSubstituteImage(original)
.asCompatibleSubstituteFor(original);

if (!result.equals(original)) {
log.warn("Image name {} was substituted by configuration to {}. This approach is deprecated and will be removed in the future",
original,
result
);
}

return result;
}

@Override
protected int getPriority() {
return -2;
}

@Override
protected String getDescription() {
return getClass().getSimpleName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

/**
* Testcontainers' default implementation of {@link ImageNameSubstitutor}.
* Delegates to {@link ConfigurationFileImageNameSubstitutor}.
* <p>
* WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader}
* requires it to be so. Public visibility DOES NOT make it part of the public API.
*/
@Slf4j
public class DefaultImageNameSubstitutor extends ImageNameSubstitutor {

private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor;

public DefaultImageNameSubstitutor() {
configurationFileImageNameSubstitutor = new ConfigurationFileImageNameSubstitutor();
}

@VisibleForTesting
DefaultImageNameSubstitutor(
final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor
) {
this.configurationFileImageNameSubstitutor = configurationFileImageNameSubstitutor;
}

@Override
public DockerImageName apply(final DockerImageName original) {
return configurationFileImageNameSubstitutor.apply(original);
}

@Override
protected int getPriority() {
return 0;
}

@Override
protected String getDescription() {
return "DefaultImageNameSubstitutor (delegates to '" + configurationFileImageNameSubstitutor.getDescription() + "')";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

import java.util.ServiceLoader;
import java.util.function.Function;
import java.util.stream.StreamSupport;

import static java.util.Comparator.comparingInt;

/**
* An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name.
* This is intended to provide a way to override image names, for example to enforce pulling of images from a private
* registry.
*/
@Slf4j
public abstract class ImageNameSubstitutor implements Function<DockerImageName, DockerImageName> {

@VisibleForTesting
static ImageNameSubstitutor instance;

public synchronized static ImageNameSubstitutor instance() {
if (instance == null) {
final ServiceLoader<ImageNameSubstitutor> serviceLoader = ServiceLoader.load(ImageNameSubstitutor.class);

instance = StreamSupport.stream(serviceLoader.spliterator(), false)
.peek(it -> log.debug("Found ImageNameSubstitutor using ServiceLoader: {} (priority {}) ", it, it.getPriority()))
.max(comparingInt(ImageNameSubstitutor::getPriority))
.map(ImageNameSubstitutor::wrapWithLogging)
.orElseThrow(() -> new RuntimeException("Unable to find any ImageNameSubstitutor using ServiceLoader"));

log.info("Using ImageNameSubstitutor: {}", instance);
}

return instance;
}

private static ImageNameSubstitutor wrapWithLogging(final ImageNameSubstitutor wrappedInstance) {
return new LogWrappedImageNameSubstitutor(wrappedInstance);
}

/**
* Substitute a {@link DockerImageName} for another, for example to replace a generic Docker Hub image name with a
* private registry copy of the image.
*
* @param original original name to be replaced
* @return a replacement name, or the original, as appropriate
*/
public abstract DockerImageName apply(DockerImageName original);

/**
* Priority of this {@link ImageNameSubstitutor} compared to other instances that may be found by the service
* loader. The highest priority instance found will always be used.
*
* @return a priority
*/
protected abstract int getPriority();

protected abstract String getDescription();

/**
* Wrapper substitutor which logs which substitutions have been performed.
*/
static class LogWrappedImageNameSubstitutor extends ImageNameSubstitutor {
@VisibleForTesting
final ImageNameSubstitutor wrappedInstance;

public LogWrappedImageNameSubstitutor(final ImageNameSubstitutor wrappedInstance) {
this.wrappedInstance = wrappedInstance;
}

@Override
public DockerImageName apply(final DockerImageName original) {
final String className = wrappedInstance.getClass().getName();
final DockerImageName replacementImage = wrappedInstance.apply(original);

if (!replacementImage.equals(original)) {
log.info("Using {} as a substitute image for {} (using image substitutor: {})", replacementImage.asCanonicalNameString(), original.asCanonicalNameString(), className);
return replacementImage;
} else {
log.debug("Did not find a substitute image for {} (using image substitutor: {})", original.asCanonicalNameString(), className);
return original;
}
}

@Override
protected int getPriority() {
return wrappedInstance.getPriority();
}

@Override
protected String getDescription() {
return wrappedInstance.getDescription();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ private ResourceReaper() {

@SneakyThrows(InterruptedException.class)
public static String start(String hostIpAddress, DockerClient client) {
String ryukImage = TestcontainersConfiguration.getInstance().getRyukDockerImageName().asCanonicalNameString();
String ryukImage = ImageNameSubstitutor.instance()
.apply(DockerImageName.parse("testcontainers/ryuk:0.3.0"))
.asCanonicalNameString();
DockerClientFactory.instance().checkAndPullImage(client, ryukImage);

List<Bind> binds = new ArrayList<>();
Expand Down

0 comments on commit 950af34

Please sign in to comment.