JUnit rule for flaky test retry

core/build.gradle

@@ -113,7 +113,7 @@ dependencies {
     testCompile files('testlib/repo/fakejar/fakejar/0/fakejar-0.jar')
 
     testCompile 'org.assertj:assertj-core:3.12.2'
-
+    testCompile project(":test-support")
 
     jarFileTestCompileOnly "org.projectlombok:lombok:${lombok.version}"
     jarFileTestCompile 'junit:junit:4.12'

core/src/test/java/org/testcontainers/junit/GenericContainerRuleTest.java

@@ -6,18 +6,36 @@
 import com.mongodb.MongoClient;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
-import com.rabbitmq.client.*;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
 import org.apache.commons.io.FileUtils;
 import org.bson.Document;
-import org.junit.*;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
 import org.rnorth.ducttape.RetryCountExceededException;
 import org.rnorth.ducttape.unreliables.Unreliables;
 import org.testcontainers.containers.Container;
 import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.testsupport.Flaky;
+import org.testcontainers.testsupport.FlakyTestJUnit4RetryRule;
 import org.testcontainers.utility.Base58;
 import org.testcontainers.utility.TestEnvironment;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
 import java.net.Socket;
 import java.time.Duration;
 import java.util.Arrays;
@@ -128,28 +146,8 @@ public static void setupContent() throws FileNotFoundException {
             .withExtraHost("somehost", "192.168.1.10")
             .withCommand("/bin/sh", "-c", "while true; do cat /etc/hosts | nc -l -p 80; done");
 
-//    @Test
-//    public void simpleRedisTest() {
-//        String ipAddress = redis.getContainerIpAddress();
-//        Integer port = redis.getMappedPort(REDIS_PORT);
-//
-//        // Use Redisson to obtain a List that is backed by Redis
-//        Config redisConfig = new Config();
-//        redisConfig.useSingleServer().setAddress(ipAddress + ":" + port);
-//
-//        Redisson redisson = Redisson.create(redisConfig);
-//
-//        List<String> testList = redisson.getList("test");
-//        testList.add("foo");
-//        testList.add("bar");
-//        testList.add("baz");
-//
-//        List<String> testList2 = redisson.getList("test");
-//        assertEquals("The list contains the expected number of items (redis is working!)", 3, testList2.size());
-//        assertTrue("The list contains an item that was put in (redis is working!)", testList2.contains("foo"));
-//        assertTrue("The list contains an item that was put in (redis is working!)", testList2.contains("bar"));
-//        assertTrue("The list contains an item that was put in (redis is working!)", testList2.contains("baz"));
-//    }
+    @Rule
+    public FlakyTestJUnit4RetryRule retry = new FlakyTestJUnit4RetryRule();
 
     @Test
     public void testIsRunning() {
@@ -402,6 +400,7 @@ public void addExposedPortAfterWithExposedPortsTest() {
     }
 
     @Test
+    @Flaky(rationale = "Shared memory seems to be unreliable in some CI environments (Travis, Azure)")
     public void sharedMemorySetTest() {
         try (GenericContainer containerWithSharedMemory = new GenericContainer()
             .withSharedMemorySize(42L * FileUtils.ONE_MB)) {

settings.gradle

@@ -41,3 +41,5 @@ file('modules').eachDir { dir ->
 
 include "docs-examples"
 project(":docs-examples").projectDir = file("docs/examples")
+include 'test-support'
+

test-support/src/main/java/org/testcontainers/testsupport/Flaky.java

@@ -0,0 +1,18 @@
+package org.testcontainers.testsupport;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.TYPE;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * TODO: Javadocs
+ */
+@Retention(RUNTIME)
+@Target({METHOD, TYPE})
+public @interface Flaky {
+
+    String rationale();
+}

test-support/src/main/java/org/testcontainers/testsupport/FlakyTestJUnit4RetryRule.java

@@ -0,0 +1,57 @@
+package org.testcontainers.testsupport;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TODO: Javadocs
+ */
+@Slf4j
+public class FlakyTestJUnit4RetryRule extends TestWatcher {
+
+    @Override
+    public Statement apply(Statement base, Description description) {
+        if (description.getAnnotation(Flaky.class) != null) {
+            return new RetryingStatement(base, description);
+        } else {
+            return super.apply(base, description);
+        }
+    }
+
+    private static class RetryingStatement extends Statement {
+        private final Statement base;
+        private final Description description;
+
+        RetryingStatement(Statement base, Description description) {
+            this.base = base;
+            this.description = description;
+        }
+
+        @Override
+        public void evaluate() {
+
+            int attempts = 0;
+            final List<Throwable> causes = new ArrayList<>();
+
+            while (++attempts < 3) {
+                try {
+                    base.evaluate();
+                    return;
+                } catch (Throwable throwable) {
+                    log.info("Retrying @Flaky-annotated test: {}", description.getDisplayName());
+                    causes.add(throwable);
+                }
+            }
+
+            throw new IllegalStateException(
+                "@Flaky-annotated test failed despite retries.",
+                new MultipleFailureException(causes));
+        }
+    }
+}

test-support/src/test/resources/logback-test.xml

@@ -0,0 +1,21 @@
+<configuration>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- encoders are assigned the type
+             ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} %-5level %logger - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="STDOUT"/>
+    </root>
+
+    <logger name="org.testcontainers" level="DEBUG"/>
+
+    <logger name="com.github.dockerjava" level="WARN"/>
+    <logger name="org.zeroturnaround.exec" level="WARN"/>
+    <logger name="org.testcontainers.shaded" level="WARN"/>
+
+</configuration>