Skip to content

Commit

Permalink
Include copyToTransferableContainerPathMap in re-use hashing
Browse files Browse the repository at this point in the history
  • Loading branch information
Alex Stockinger committed Apr 7, 2022
1 parent 82eb3ca commit 4f28875
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 39 deletions.
38 changes: 15 additions & 23 deletions core/src/main/java/org/testcontainers/containers/GenericContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,6 @@ public class GenericContainer<SELF extends GenericContainer<SELF>>
private Map<MountableFile, String> copyToFileContainerPathMap = new LinkedHashMap<>();

// Maintain order in which entries are added, as earlier target location may be a prefix of a later location.
@Getter(AccessLevel.NONE)
@Setter(AccessLevel.NONE)
private Map<Transferable, String> copyToTransferableContainerPathMap = new LinkedHashMap<>();

Expand Down Expand Up @@ -540,33 +539,21 @@ private void tryStart(Instant startedAt) {
@VisibleForTesting
Checksum hashCopiedFiles() {
Checksum checksum = new Adler32();
final Consumer<String> updateFromString = string -> {
byte[] bytes = string.getBytes();
checksum.update(bytes, 0, bytes.length);
};
copyToFileContainerPathMap.entrySet().stream().sorted(Entry.comparingByValue()).forEach(entry -> {
byte[] pathBytes = entry.getValue().getBytes();
// Add path to the hash
checksum.update(pathBytes, 0, pathBytes.length);

File file = new File(entry.getKey().getResolvedPath());
checksumFile(file, checksum);
updateFromString.accept(entry.getValue());
entry.getKey().updateChecksum(checksum);
});
copyToTransferableContainerPathMap.entrySet().stream().sorted(Entry.comparingByValue()).forEach(entry -> {
updateFromString.accept(entry.getValue());
entry.getKey().updateChecksum(checksum);
});
return checksum;
}

@VisibleForTesting
@SneakyThrows(IOException.class)
void checksumFile(File file, Checksum checksum) {
Path path = file.toPath();
checksum.update(MountableFile.getUnixFileMode(path));
if (file.isDirectory()) {
try (Stream<Path> stream = Files.walk(path)) {
stream.filter(it -> it != path).forEach(it -> {
checksumFile(it.toFile(), checksum);
});
}
} else {
FileUtils.checksum(file, checksum);
}
}

@UnstableAPI
@SneakyThrows(JsonProcessingException.class)
final String hash(CreateContainerCmd createCommand) {
Expand Down Expand Up @@ -1475,4 +1462,9 @@ public int hashCode() {
public String getContainerName() {
return getContainerInfo().getName();
}

@VisibleForTesting
Map<Transferable, String> getCopyToTransferableContainerPathMap() {
return copyToTransferableContainerPathMap;
}
}
11 changes: 11 additions & 0 deletions core/src/main/java/org/testcontainers/images/builder/Transferable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.zip.Checksum;

public interface Transferable {

Expand All @@ -32,6 +33,12 @@ public byte[] getBytes() {
return bytes;
}

@Override
public void updateChecksum(Checksum checksum) {
byte[] bytes = getBytes();
checksum.update(bytes, 0, bytes.length);
}

@Override
public int getFileMode() {
return fileMode;
Expand Down Expand Up @@ -83,4 +90,8 @@ default byte[] getBytes() {
default String getDescription() {
return "";
}

default void updateChecksum(Checksum checksum) {
throw new UnsupportedOperationException("Provide implementation in subclass");
}
}
30 changes: 30 additions & 0 deletions core/src/main/java/org/testcontainers/utility/MountableFile.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.tar.TarConstants;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
import org.jetbrains.annotations.NotNull;
import org.testcontainers.DockerClientFactory;
Expand All @@ -28,6 +31,8 @@
import java.util.Set;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Stream;
import java.util.zip.Checksum;

import static lombok.AccessLevel.PACKAGE;
import static org.testcontainers.utility.PathUtils.recursiveDeleteDir;
Expand Down Expand Up @@ -361,6 +366,31 @@ public String getDescription() {
return this.getResolvedPath();
}

@Override
public void updateChecksum(Checksum checksum) {
byte[] pathBytes = getBytes();
// Add path to the hash
checksum.update(pathBytes, 0, pathBytes.length);

File file = new File(getResolvedPath());
checksumFile(file, checksum);
}

@SneakyThrows(IOException.class)
private void checksumFile(File file, Checksum checksum) {
Path path = file.toPath();
checksum.update(MountableFile.getUnixFileMode(path));
if (file.isDirectory()) {
try (Stream<Path> stream = Files.walk(path)) {
stream.filter(it -> it != path).forEach(it -> {
checksumFile(it.toFile(), checksum);
});
}
} else {
FileUtils.checksum(file, checksum);
}
}

@Override
public int getFileMode() {
return getUnixFileMode(this.getResolvedPath());
Expand Down
87 changes: 71 additions & 16 deletions core/src/test/java/org/testcontainers/containers/ReusabilityUnitTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import com.github.dockerjava.api.command.ListContainersCmd;
import com.github.dockerjava.api.command.StartContainerCmd;
import com.github.dockerjava.api.model.Container;
import com.github.dockerjava.api.model.NetworkSettings;
import com.github.dockerjava.core.command.CreateContainerCmdImpl;
import com.github.dockerjava.core.command.InspectContainerCmdImpl;
import com.github.dockerjava.core.command.ListContainersCmdImpl;
Expand All @@ -24,13 +23,13 @@
import org.junit.runners.BlockJUnit4ClassRunner;
import org.junit.runners.Parameterized;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
import org.rnorth.visibleassertions.VisibleAssertions;
import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.startupcheck.StartupCheckStrategy;
import org.testcontainers.containers.wait.strategy.AbstractWaitStrategy;
import org.testcontainers.images.builder.Transferable;
import org.testcontainers.utility.MockTestcontainersConfigurationRule;
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.TestcontainersConfiguration;
Expand All @@ -39,12 +38,14 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -297,11 +298,65 @@ public void shouldHashCopiedFiles() {
}
}

@RunWith(BlockJUnit4ClassRunner.class)
@RunWith(Parameterized.class)
@FieldDefaults(makeFinal = true)
public static class CopyFilesHashTest {
private final TestStrategy strategy;

interface TestStrategy {
void withCopyFileToContainer(MountableFile mountableFile, String path);

void clear();
}

private static class MountableFileTestStrategy implements TestStrategy {
private final GenericContainer<?> container;

private MountableFileTestStrategy(GenericContainer<?> container) {
this.container = container;
}

@Override
public void withCopyFileToContainer(MountableFile mountableFile, String path) {
container.withCopyFileToContainer(mountableFile, path);
}

@Override
public void clear() {
container.getCopyToFileContainerPathMap().clear();
}
}

private static class TransferableTestStrategy implements TestStrategy {
private final GenericContainer<?> container;

private TransferableTestStrategy(GenericContainer<?> container) {
this.container = container;
}

@Override
public void withCopyFileToContainer(MountableFile mountableFile, String path) {
container.withCopyFileToContainer((Transferable)mountableFile, path);
}

@Override
public void clear() {
container.getCopyToTransferableContainerPathMap().clear();
}
}

@Parameterized.Parameters
public static List<Function<GenericContainer<?>, TestStrategy>> strategies() {
return Arrays.asList(MountableFileTestStrategy::new, TransferableTestStrategy::new);
}


GenericContainer<?> container = new GenericContainer<>(TINY_IMAGE);

public CopyFilesHashTest(Function<GenericContainer<?>, TestStrategy> strategyFactory) {
this.strategy = strategyFactory.apply(container);
}

@Test
public void empty() {
assertThat(container.hashCopiedFiles()).isNotNull();
Expand All @@ -311,7 +366,7 @@ public void empty() {
public void oneFile() {
long emptyHash = container.hashCopiedFiles().getValue();

container.withCopyFileToContainer(
strategy.withCopyFileToContainer(
MountableFile.forClasspathResource("test_copy_to_container.txt"),
"/foo/bar"
);
Expand All @@ -322,13 +377,13 @@ public void oneFile() {
@Test
public void differentPath() {
MountableFile mountableFile = MountableFile.forClasspathResource("test_copy_to_container.txt");
container.withCopyFileToContainer(mountableFile, "/foo/bar");
strategy.withCopyFileToContainer(mountableFile, "/foo/bar");

long hash1 = container.hashCopiedFiles().getValue();

container.getCopyToFileContainerPathMap().clear();
strategy.clear();

container.withCopyFileToContainer(mountableFile, "/foo/baz");
strategy.withCopyFileToContainer(mountableFile, "/foo/baz");

assertThat(container.hashCopiedFiles().getValue()).isNotEqualTo(hash1);
}
Expand All @@ -337,7 +392,7 @@ public void differentPath() {
public void detectsChangesInFile() throws Exception {
Path path = File.createTempFile("reusable_test", ".txt").toPath();
MountableFile mountableFile = MountableFile.forHostPath(path);
container.withCopyFileToContainer(mountableFile, "/foo/bar");
strategy.withCopyFileToContainer(mountableFile, "/foo/bar");

long hash1 = container.hashCopiedFiles().getValue();

Expand All @@ -348,13 +403,13 @@ public void detectsChangesInFile() throws Exception {

@Test
public void multipleFiles() {
container.withCopyFileToContainer(
strategy.withCopyFileToContainer(
MountableFile.forClasspathResource("test_copy_to_container.txt"),
"/foo/bar"
);
long hash1 = container.hashCopiedFiles().getValue();

container.withCopyFileToContainer(
strategy.withCopyFileToContainer(
MountableFile.forClasspathResource("mappable-resource/test-resource.txt"),
"/foo/baz"
);
Expand All @@ -368,7 +423,7 @@ public void folder() throws Exception {

Path tempDirectory = Files.createTempDirectory("reusable_test");
MountableFile mountableFile = MountableFile.forHostPath(tempDirectory);
container.withCopyFileToContainer(mountableFile, "/foo/bar/");
strategy.withCopyFileToContainer(mountableFile, "/foo/bar/");

assertThat(container.hashCopiedFiles().getValue()).isNotEqualTo(emptyHash);
}
Expand All @@ -378,7 +433,7 @@ public void changesInFolder() throws Exception {
Path tempDirectory = Files.createTempDirectory("reusable_test");
MountableFile mountableFile = MountableFile.forHostPath(tempDirectory);
assertThat(new File(mountableFile.getResolvedPath())).isDirectory();
container.withCopyFileToContainer(mountableFile, "/foo/bar/");
strategy.withCopyFileToContainer(mountableFile, "/foo/bar/");

long hash1 = container.hashCopiedFiles().getValue();

Expand All @@ -397,11 +452,11 @@ public void folderAndFile() throws Exception {
Path tempDirectory = Files.createTempDirectory("reusable_test");
MountableFile mountableFile = MountableFile.forHostPath(tempDirectory);
assertThat(new File(mountableFile.getResolvedPath())).isDirectory();
container.withCopyFileToContainer(mountableFile, "/foo/bar/");
strategy.withCopyFileToContainer(mountableFile, "/foo/bar/");

long hash1 = container.hashCopiedFiles().getValue();

container.withCopyFileToContainer(
strategy.withCopyFileToContainer(
MountableFile.forClasspathResource("test_copy_to_container.txt"),
"/foo/baz"
);
Expand All @@ -414,7 +469,7 @@ public void filePermissions() throws Exception {
Path path = File.createTempFile("reusable_test", ".txt").toPath();
path.toFile().setExecutable(false);
MountableFile mountableFile = MountableFile.forHostPath(path);
container.withCopyFileToContainer(mountableFile, "/foo/bar");
strategy.withCopyFileToContainer(mountableFile, "/foo/bar");

long hash1 = container.hashCopiedFiles().getValue();

Expand All @@ -432,7 +487,7 @@ public void folderPermissions() throws Exception {
Path subDir = Files.createDirectory(tempDirectory.resolve("sub"));
subDir.toFile().setWritable(false);
assumeThat(subDir.toFile().canWrite()).isFalse();
container.withCopyFileToContainer(mountableFile, "/foo/bar/");
strategy.withCopyFileToContainer(mountableFile, "/foo/bar/");

long hash1 = container.hashCopiedFiles().getValue();

Expand Down

0 comments on commit 4f28875

Please sign in to comment.