Skip to content

Commit

Permalink
Prune from JVM hook if Ryuk is disabled (#4960)
Browse files Browse the repository at this point in the history 
If Ryuk is disabled, singleton containers and other uncontrolled containers were remaining dangling.
This change adds best-effort Ryuk-like functionality that will use a JVM hook to clean by labels.

Co-authored-by: Richard North <rich.north@gmail.com>
  • Loading branch information
@bsideup @rnorth
bsideup and rnorth committed Jan 31, 2022
1 parent 242dfde commit 205f21c
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 24 deletions.
3 changes: 3 additions & 0 deletions core/src/main/java/org/testcontainers/DockerClientFactory.java
View file
Expand Up @@ -225,6 +225,9 @@ public void close() {
} else {
log.debug("Ryuk is disabled");
ryukContainerId = null;
// best-efforts cleanup at JVM shutdown, without using the Ryuk container
//noinspection deprecation
ResourceReaper.instance().setHook();
}

boolean checksEnabled = !TestcontainersConfiguration.getInstance().isDisableChecks();
Expand Down
6 changes: 2 additions & 4 deletions core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java
View file
Expand Up @@ -41,7 +41,6 @@

import java.io.File;
import java.time.Duration;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -56,7 +55,6 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static com.google.common.base.Preconditions.checkArgument;
Expand Down Expand Up @@ -310,8 +308,8 @@ private void runWithCompose(String cmd) {
}

private void registerContainersForShutdown() {
ResourceReaper.instance().registerFilterForCleanup(Arrays.asList(
new SimpleEntry<>("label", "com.docker.compose.project=" + project)
ResourceReaper.instance().registerLabelsFilterForCleanup(Collections.singletonMap(
"com.docker.compose.project", project
));
}

Expand Down
97 changes: 77 additions & 20 deletions core/src/main/java/org/testcontainers/utility/ResourceReaper.java
View file
Expand Up @@ -5,12 +5,14 @@
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Container;
import com.github.dockerjava.api.model.ExposedPort;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.HostConfig;
import com.github.dockerjava.api.model.Network;
import com.github.dockerjava.api.model.PortBinding;
import com.github.dockerjava.api.model.Ports;
import com.github.dockerjava.api.model.PruneType;
import com.github.dockerjava.api.model.Volume;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
Expand All @@ -36,6 +38,7 @@
import java.nio.charset.StandardCharsets;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
Expand All @@ -58,24 +61,28 @@ public final class ResourceReaper {

private static final Logger LOGGER = LoggerFactory.getLogger(ResourceReaper.class);

private static final List<List<Map.Entry<String, String>>> DEATH_NOTE = new ArrayList<>();
private static final List<List<Map.Entry<String, String>>> DEATH_NOTE = new ArrayList<>(
Arrays.asList(
DockerClientFactory.DEFAULT_LABELS.entrySet().stream()
.<Map.Entry<String, String>>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue()))
.collect(Collectors.toList())
)
);

private static final RateLimiter RYUK_ACK_RATE_LIMITER = RateLimiterBuilder
.newBuilder()
.withRate(4, TimeUnit.SECONDS)
.withConstantThroughput()
.build();

private static ResourceReaper instance;
private final DockerClient dockerClient;
private static AtomicBoolean ryukStarted = new AtomicBoolean(false);
private final DockerClient dockerClient = DockerClientFactory.lazyClient();
private Map<String, String> registeredContainers = new ConcurrentHashMap<>();
private Set<String> registeredNetworks = Sets.newConcurrentHashSet();
private Set<String> registeredImages = Sets.newConcurrentHashSet();
private AtomicBoolean hookIsSet = new AtomicBoolean(false);

private ResourceReaper() {
dockerClient = DockerClientFactory.instance().client();
}


/**
*
Expand Down Expand Up @@ -173,14 +180,6 @@ public InspectContainerResponse getContainerInfo() {

CountDownLatch ryukScheduledLatch = new CountDownLatch(1);

synchronized (DEATH_NOTE) {
DEATH_NOTE.add(
DockerClientFactory.DEFAULT_LABELS.entrySet().stream()
.<Map.Entry<String, String>>map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue()))
.collect(Collectors.toList())
);
}

String host = containerState.getHost();
Integer ryukPort = containerState.getFirstMappedPort();
Thread kiraThread = new Thread(
Expand Down Expand Up @@ -238,6 +237,7 @@ public InspectContainerResponse getContainerInfo() {
}
}

ryukStarted.set(true);
return ryukContainerId;
}

Expand All @@ -253,7 +253,7 @@ public synchronized static ResourceReaper instance() {
* Perform a cleanup.
*/
public synchronized void performCleanup() {
registeredContainers.forEach(this::stopContainer);
registeredContainers.forEach(this::removeContainer);
registeredNetworks.forEach(this::removeNetwork);
registeredImages.forEach(this::removeImage);
}
Expand All @@ -262,14 +262,29 @@ public synchronized void performCleanup() {
* Register a filter to be cleaned up.
*
* @param filter the filter
* @deprecated only label filter is supported by the prune API, use {@link #registerLabelsFilterForCleanup(Map)}
*/
@Deprecated
public void registerFilterForCleanup(List<Map.Entry<String, String>> filter) {
synchronized (DEATH_NOTE) {
DEATH_NOTE.add(filter);
DEATH_NOTE.notifyAll();
}
}

/**
* Register a label to be cleaned up.
*
* @param labels the filter
*/
public void registerLabelsFilterForCleanup(Map<String, String> labels) {
registerFilterForCleanup(
labels.entrySet().stream()
.map(it -> new SimpleEntry<>("label", it.getKey() + "=" + it.getValue()))
.collect(Collectors.toList())
);
}

/**
* Register a container to be cleaned up, either on explicit call to stopAndRemoveContainer, or at JVM shutdown.
*
Expand All @@ -287,7 +302,7 @@ public void registerContainerForCleanup(String containerId, String imageName) {
* @param containerId the ID of the container
*/
public void stopAndRemoveContainer(String containerId) {
stopContainer(containerId, registeredContainers.get(containerId));
removeContainer(containerId, registeredContainers.get(containerId));

registeredContainers.remove(containerId);
}
Expand All @@ -299,12 +314,12 @@ public void stopAndRemoveContainer(String containerId) {
* @param imageName the image name of the container (used for logging)
*/
public void stopAndRemoveContainer(String containerId, String imageName) {
stopContainer(containerId, imageName);
removeContainer(containerId, imageName);

registeredContainers.remove(containerId);
}

private void stopContainer(String containerId, String imageName) {
private void removeContainer(String containerId, String imageName) {
boolean running;
try {
InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerId).exec();
Expand Down Expand Up @@ -444,10 +459,52 @@ private void removeImage(String dockerImageName) {
}
}

private void setHook() {
private void prune(PruneType pruneType, List<Map.Entry<String, String>> filters) {
String[] labels = filters.stream()
.filter(it -> "label".equals(it.getKey()))
.map(Map.Entry::getValue)
.toArray(String[]::new);
switch (pruneType) {
// Docker only prunes stopped containers, so we have to do it manually
case CONTAINERS:
List<Container> containers = dockerClient.listContainersCmd()
.withFilter("label", Arrays.asList(labels))
.withShowAll(true)
.exec();

containers.parallelStream().forEach(container -> {
removeContainer(container.getId(), container.getImage());
});
break;
default:
dockerClient.pruneCmd(pruneType).withLabelFilter(labels).exec();
break;
}
}

/**
* @deprecated internal API, not intended for public usage
*/
@Deprecated
public void setHook() {
if (hookIsSet.compareAndSet(false, true)) {
// If the JVM stops without containers being stopped, try and stop the container.
Runtime.getRuntime().addShutdownHook(new Thread(DockerClientFactory.TESTCONTAINERS_THREAD_GROUP, this::performCleanup));
Runtime.getRuntime().addShutdownHook(
new Thread(DockerClientFactory.TESTCONTAINERS_THREAD_GROUP,
() -> {
performCleanup();

if (!ryukStarted.get()) {
synchronized (DEATH_NOTE) {
DEATH_NOTE.forEach(filters -> prune(PruneType.CONTAINERS, filters));
DEATH_NOTE.forEach(filters -> prune(PruneType.NETWORKS, filters));
DEATH_NOTE.forEach(filters -> prune(PruneType.VOLUMES, filters));
DEATH_NOTE.forEach(filters -> prune(PruneType.IMAGES, filters));
}
}
}
)
);
}
}

Expand Down

0 comments on commit 205f21c

Please sign in to comment.