JUnit rule for flaky test retry

core/build.gradle

@@ -113,7 +113,7 @@ dependencies {
     testCompile files('testlib/repo/fakejar/fakejar/0/fakejar-0.jar')
 
     testCompile 'org.assertj:assertj-core:3.12.2'
-
+    testCompile project(':test-support')
 
     jarFileTestCompileOnly "org.projectlombok:lombok:${lombok.version}"
     jarFileTestCompile 'junit:junit:4.12'

core/src/test/java/org/testcontainers/junit/GenericContainerRuleTest.java

@@ -6,18 +6,36 @@
 import com.mongodb.MongoClient;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
-import com.rabbitmq.client.*;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
 import org.apache.commons.io.FileUtils;
 import org.bson.Document;
-import org.junit.*;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
 import org.rnorth.ducttape.RetryCountExceededException;
 import org.rnorth.ducttape.unreliables.Unreliables;
 import org.testcontainers.containers.Container;
 import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.testsupport.Flaky;
+import org.testcontainers.testsupport.FlakyTestJUnit4RetryRule;
 import org.testcontainers.utility.Base58;
 import org.testcontainers.utility.TestEnvironment;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
 import java.net.Socket;
 import java.time.Duration;
 import java.util.Arrays;
@@ -128,28 +146,8 @@ public static void setupContent() throws FileNotFoundException {
             .withExtraHost("somehost", "192.168.1.10")
             .withCommand("/bin/sh", "-c", "while true; do cat /etc/hosts | nc -l -p 80; done");
 
-//    @Test
-//    public void simpleRedisTest() {
-//        String ipAddress = redis.getContainerIpAddress();
-//        Integer port = redis.getMappedPort(REDIS_PORT);
-//
-//        // Use Redisson to obtain a List that is backed by Redis
-//        Config redisConfig = new Config();
-//        redisConfig.useSingleServer().setAddress(ipAddress + ":" + port);
-//
-//        Redisson redisson = Redisson.create(redisConfig);
-//
-//        List<String> testList = redisson.getList("test");
-//        testList.add("foo");
-//        testList.add("bar");
-//        testList.add("baz");
-//
-//        List<String> testList2 = redisson.getList("test");
-//        assertEquals("The list contains the expected number of items (redis is working!)", 3, testList2.size());
-//        assertTrue("The list contains an item that was put in (redis is working!)", testList2.contains("foo"));
-//        assertTrue("The list contains an item that was put in (redis is working!)", testList2.contains("bar"));
-//        assertTrue("The list contains an item that was put in (redis is working!)", testList2.contains("baz"));
-//    }
+    @Rule
+    public FlakyTestJUnit4RetryRule retry = new FlakyTestJUnit4RetryRule();
 
     @Test
     public void testIsRunning() {
@@ -402,6 +400,7 @@ public void addExposedPortAfterWithExposedPortsTest() {
     }
 
     @Test
+    @Flaky(githubIssueUrl = "https://github.com/testcontainers/testcontainers-java/issues/1688", reviewDate = "2019-10-01")
     public void sharedMemorySetTest() {
         try (GenericContainer containerWithSharedMemory = new GenericContainer()
             .withSharedMemorySize(42L * FileUtils.ONE_MB)) {

modules/couchbase/build.gradle

@@ -3,4 +3,6 @@ description = "Testcontainers :: Couchbase"
 dependencies {
     compile project(':testcontainers')
     compile 'com.couchbase.client:java-client:2.7.7'
+
+    testCompile project(':test-support')
 }

modules/couchbase/src/test/java/org/testcontainers/couchbase/BaseCouchbaseContainerTest.java

@@ -9,7 +9,10 @@
 import com.couchbase.client.java.view.View;
 import com.google.common.collect.Lists;
 import org.junit.Assert;
+import org.junit.Rule;
 import org.junit.Test;
+import org.testcontainers.testsupport.Flaky;
+import org.testcontainers.testsupport.FlakyTestJUnit4RetryRule;
 
 import java.util.List;
 
@@ -25,7 +28,11 @@ public abstract class BaseCouchbaseContainerTest extends AbstractCouchbaseTest {
 
     private static final String DOCUMENT = "{\"name\":\"toto\"}";
 
+    @Rule
+    public FlakyTestJUnit4RetryRule retry = new FlakyTestJUnit4RetryRule();
+
     @Test
+    @Flaky(githubIssueUrl = "https://github.com/testcontainers/testcontainers-java/issues/1453", reviewDate = "2019-10-01") // TODO: describe
     public void shouldInsertDocument() {
         RawJsonDocument expected = RawJsonDocument.create(ID, DOCUMENT);
         getBucket().upsert(expected);
@@ -34,6 +41,7 @@ public void shouldInsertDocument() {
     }
 
     @Test
+    @Flaky(githubIssueUrl = "https://github.com/testcontainers/testcontainers-java/issues/1453", reviewDate = "2019-10-01") // TODO: describe
     public void shouldExecuteN1ql() {
         getBucket().query(N1qlQuery.simple("INSERT INTO " + TEST_BUCKET + " (KEY, VALUE) VALUES ('" + ID + "', " + DOCUMENT + ")"));
 
@@ -46,6 +54,7 @@ public void shouldExecuteN1ql() {
     }
 
     @Test
+    @Flaky(githubIssueUrl = "https://github.com/testcontainers/testcontainers-java/issues/1453", reviewDate = "2019-10-01") // TODO: describe
     public void shouldCreateView() {
         View view = DefaultView.create(VIEW_NAME, VIEW_FUNCTION);
         DesignDocument document = DesignDocument.create(VIEW_NAME, Lists.newArrayList(view));

modules/jdbc-test/build.gradle

@@ -28,4 +28,5 @@ dependencies {
     testCompile 'commons-dbutils:commons-dbutils:1.6'
 
     testCompile 'com.googlecode.junit-toolbox:junit-toolbox:2.4'
+    testCompile project(':test-support')
 }

modules/jdbc-test/src/test/java/org/testcontainers/jdbc/DatabaseDriverTmpfsTest.java

@@ -1,18 +1,26 @@
 package org.testcontainers.jdbc;
 
+import org.junit.Rule;
+import org.junit.Test;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.JdbcDatabaseContainer;
+import org.testcontainers.testsupport.Flaky;
+import org.testcontainers.testsupport.FlakyTestJUnit4RetryRule;
+
 import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
-import org.junit.Test;
-import org.testcontainers.containers.Container;
-import org.testcontainers.containers.JdbcDatabaseContainer;
 
 import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;
 
 public class DatabaseDriverTmpfsTest {
 
+    @Rule
+    public FlakyTestJUnit4RetryRule retry = new FlakyTestJUnit4RetryRule();
+
     @Test
+    @Flaky(githubIssueUrl = "https://github.com/testcontainers/testcontainers-java/issues/1687", reviewDate = "2019-10-01") // TODO: describe
     public void tmpfs() throws IOException, InterruptedException, SQLException {
         final String jdbcUrl = "jdbc:tc:postgresql:9.6.8://hostname/databasename?TC_TMPFS=/testtmpfs:rw";
         try (Connection ignored = DriverManager.getConnection(jdbcUrl)) {

settings.gradle

@@ -41,3 +41,5 @@ file('modules').eachDir { dir ->
 
 include "docs-examples"
 project(":docs-examples").projectDir = file("docs/examples")
+include 'test-support'
+

test-support/build.gradle

@@ -0,0 +1,4 @@
+dependencies {
+    implementation 'junit:junit:4.12'
+    implementation 'org.slf4j:slf4j-api:1.7.26'
+}

test-support/src/main/java/org/testcontainers/testsupport/Flaky.java

@@ -0,0 +1,28 @@
+package org.testcontainers.testsupport;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * TODO: Javadocs
+ */
+@Retention(RUNTIME)
+@Target({METHOD})
+public @interface Flaky {
+
+    /**
+     * @return a URL for a GitHub issue where this flaky test can be discussed, and where actions to resolve it can be
+     * coordinated.
+     */
+    String githubIssueUrl();
+
+    /**
+     * @return a date at which this should be reviewed, in {@link java.time.format.DateTimeFormatter#ISO_LOCAL_DATE}
+     * format (e.g. {@code 2020-12-03}). Now + 3 months is suggested. Once this date has passed, the annotation will
+     * stop having an effect.
+     */
+    String reviewDate();
+}

test-support/src/main/java/org/testcontainers/testsupport/FlakyTestJUnit4RetryRule.java

@@ -0,0 +1,72 @@
+package org.testcontainers.testsupport;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+
+import java.time.LocalDate;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TODO: Javadocs
+ */
+@Slf4j
+public class FlakyTestJUnit4RetryRule implements TestRule {
+
+    @Override
+    public Statement apply(Statement base, Description description) {
+        final Flaky annotation = description.getAnnotation(Flaky.class);
+        if (annotation != null) {
+            if ( ! isReviewDatePassed(annotation)) {
+                return new RetryingStatement(base, description);
+            }
+        }
+
+        return new Statement() {
+            @Override
+            public void evaluate() throws Throwable {
+                base.evaluate();
+            }
+        };
+    }
+
+    private boolean isReviewDatePassed(Flaky annotation) {
+        final LocalDate reviewDate = LocalDate.parse(annotation.reviewDate());
+
+        return reviewDate.isBefore(LocalDate.now());
+    }
+
+    private static class RetryingStatement extends Statement {
+        private final Statement base;
+        private final Description description;
+
+        RetryingStatement(Statement base, Description description) {
+            this.base = base;
+            this.description = description;
+        }
+
+        @Override
+        public void evaluate() {
+
+            int attempts = 0;
+            final List<Throwable> causes = new ArrayList<>();
+
+            while (++attempts <= 3) {
+                try {
+                    base.evaluate();
+                    return;
+                } catch (Throwable throwable) {
+                    log.info("Retrying @Flaky-annotated test: {}", description.getDisplayName());
+                    causes.add(throwable);
+                }
+            }
+
+            throw new IllegalStateException(
+                "@Flaky-annotated test failed despite retries.",
+                new MultipleFailureException(causes));
+        }
+    }
+}