Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
JUnit rule for flaky test retry (#1680)
- Loading branch information
Showing
13 changed files
with
394 additions
and
46 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
14 changes: 11 additions & 3 deletions
14
modules/jdbc-test/src/test/java/org/testcontainers/jdbc/DatabaseDriverTmpfsTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,4 @@ | ||
| dependencies { | ||
| implementation 'junit:junit:4.12' | ||
| implementation 'org.slf4j:slf4j-api:1.7.26' | ||
| } |
34 changes: 34 additions & 0 deletions
34
test-support/src/main/java/org/testcontainers/testsupport/Flaky.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| package org.testcontainers.testsupport; | ||
|
|
||
| import java.lang.annotation.Retention; | ||
| import java.lang.annotation.Target; | ||
|
|
||
| import static java.lang.annotation.ElementType.METHOD; | ||
| import static java.lang.annotation.RetentionPolicy.RUNTIME; | ||
|
|
||
| /** | ||
| * Annotation for test methods that should be retried in the event of failure. See {@link FlakyTestJUnit4RetryRule} for | ||
| * more details. | ||
| */ | ||
| @Retention(RUNTIME) | ||
| @Target({METHOD}) | ||
| public @interface Flaky { | ||
|
|
||
| /** | ||
| * @return a URL for a GitHub issue where this flaky test can be discussed, and where actions to resolve it can be | ||
| * coordinated. | ||
| */ | ||
| String githubIssueUrl(); | ||
|
|
||
| /** | ||
| * @return a date at which this should be reviewed, in {@link java.time.format.DateTimeFormatter#ISO_LOCAL_DATE} | ||
| * format (e.g. {@code 2020-12-03}). Now + 3 months is suggested. Once this date has passed, retries will no longer | ||
| * be applied. | ||
| */ | ||
| String reviewDate(); | ||
|
|
||
| /** | ||
| * @return the total number of times to try running this test (default 3) | ||
| */ | ||
| int maxTries() default 3; | ||
| } |
99 changes: 99 additions & 0 deletions
99
test-support/src/main/java/org/testcontainers/testsupport/FlakyTestJUnit4RetryRule.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| package org.testcontainers.testsupport; | ||
|
|
||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.junit.rules.TestRule; | ||
| import org.junit.runner.Description; | ||
| import org.junit.runners.model.MultipleFailureException; | ||
| import org.junit.runners.model.Statement; | ||
|
|
||
| import java.time.LocalDate; | ||
| import java.time.format.DateTimeParseException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| /** | ||
| * <p> | ||
| * JUnit 4 @Rule that implements retry for flaky tests (tests that suffer from sporadic random failures). | ||
| * </p> | ||
| * <p> | ||
| * This rule should be used in conjunction with the @{@link Flaky} annotation. When this Rule is applied to a test | ||
| * class, any test method with this annotation will be invoked up to 3 times or until it succeeds. | ||
| * </p> | ||
| * <p> | ||
| * Tests should <em>not</em> be marked @{@link Flaky} for a long period of time. Every usage should be | ||
| * accompanied by a GitHub issue URL, and should be subject to review at a suitable point in the (near) future. | ||
| * Should the review date pass without the test's instability being fixed, the retry behaviour will cease to have an | ||
| * effect and the test will be allowed to sporadically fail again. | ||
| * </p> | ||
| */ | ||
| @Slf4j | ||
| public class FlakyTestJUnit4RetryRule implements TestRule { | ||
|
|
||
| @Override | ||
| public Statement apply(Statement base, Description description) { | ||
|
|
||
| final Flaky annotation = description.getAnnotation(Flaky.class); | ||
|
|
||
| if (annotation == null) { | ||
| // leave the statement as-is | ||
| return base; | ||
| } | ||
|
|
||
| if (annotation.githubIssueUrl().trim().length() == 0) { | ||
| throw new IllegalArgumentException("A GitHub issue URL must be set for usages of the @Flaky annotation"); | ||
| } | ||
|
|
||
| final int maxTries = annotation.maxTries(); | ||
|
|
||
| if (maxTries < 1) { | ||
| throw new IllegalArgumentException("@Flaky annotation maxTries must be at least one"); | ||
| } | ||
|
|
||
| final LocalDate reviewDate; | ||
| try { | ||
| reviewDate = LocalDate.parse(annotation.reviewDate()); | ||
| } catch (DateTimeParseException e) { | ||
| throw new IllegalArgumentException("@Flaky reviewDate could not be parsed. Please provide a date in yyyy-mm-dd format"); | ||
| } | ||
|
|
||
| // the annotation should only have an effect before the review date, to encourage review and resolution | ||
| if ( LocalDate.now().isBefore(reviewDate) ) { | ||
| return new RetryingStatement(base, description, maxTries); | ||
| } else { | ||
| return base; | ||
| } | ||
| } | ||
|
|
||
| private static class RetryingStatement extends Statement { | ||
| private final Statement base; | ||
| private final Description description; | ||
| private final int maxTries; | ||
|
|
||
| RetryingStatement(Statement base, Description description, int maxTries) { | ||
| this.base = base; | ||
| this.description = description; | ||
| this.maxTries = maxTries; | ||
| } | ||
|
|
||
| @Override | ||
| public void evaluate() { | ||
|
|
||
| int attempts = 0; | ||
| final List<Throwable> causes = new ArrayList<>(); | ||
|
|
||
| while (++attempts <= maxTries) { | ||
| try { | ||
| base.evaluate(); | ||
| return; | ||
| } catch (Throwable throwable) { | ||
| log.warn("Retrying @Flaky-annotated test: {}", description.getDisplayName()); | ||
| causes.add(throwable); | ||
| } | ||
| } | ||
|
|
||
| throw new IllegalStateException( | ||
| "@Flaky-annotated test failed despite retries.", | ||
| new MultipleFailureException(causes)); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.