New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
JUnit rule for flaky test retry #1680
Changes from 17 commits
2b226e2
b65b8e5
533dde7
dfd3394
9b4891b
19b7af2
f725451
f398fde
379a673
f69dfc6
5f92e12
f79a89a
ce32d95
5405bf3
df65ac4
a3aea2f
bb6a60e
1ab2726
a904474
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
dependencies { | ||
implementation 'junit:junit:4.12' | ||
implementation 'org.slf4j:slf4j-api:1.7.26' | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
package org.testcontainers.testsupport; | ||
|
||
import java.lang.annotation.Retention; | ||
import java.lang.annotation.Target; | ||
|
||
import static java.lang.annotation.ElementType.METHOD; | ||
import static java.lang.annotation.RetentionPolicy.RUNTIME; | ||
|
||
/** | ||
* Annotation for test methods that should be retried in the event of failure. See {@link FlakyTestJUnit4RetryRule} for | ||
* more details. | ||
*/ | ||
@Retention(RUNTIME) | ||
@Target({METHOD}) | ||
public @interface Flaky { | ||
|
||
/** | ||
* @return a URL for a GitHub issue where this flaky test can be discussed, and where actions to resolve it can be | ||
* coordinated. | ||
*/ | ||
String githubIssueUrl(); | ||
|
||
/** | ||
* @return a date at which this should be reviewed, in {@link java.time.format.DateTimeFormatter#ISO_LOCAL_DATE} | ||
* format (e.g. {@code 2020-12-03}). Now + 3 months is suggested. Once this date has passed, retries will no longer | ||
* be applied. | ||
*/ | ||
String reviewDate(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. WDYT about letting them fail explicitly instead of not retrying? Or does this create too much maintenance? I think the maintenance should be the same, just without explicitly failing, we will miss the flaky tests for some time. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suspect it could end up being a PITA for contributors, to be honest! |
||
|
||
/** | ||
* @return the total number of times to try running this test (default 3) | ||
*/ | ||
int maxTries() default 3; | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
package org.testcontainers.testsupport; | ||
|
||
import lombok.extern.slf4j.Slf4j; | ||
import org.junit.rules.TestRule; | ||
import org.junit.runner.Description; | ||
import org.junit.runners.model.MultipleFailureException; | ||
import org.junit.runners.model.Statement; | ||
|
||
import java.time.LocalDate; | ||
import java.time.format.DateTimeParseException; | ||
import java.util.ArrayList; | ||
import java.util.List; | ||
|
||
/** | ||
* <p> | ||
* JUnit 4 @Rule that implements retry for flaky tests (tests that suffer from sporadic random failures). | ||
* </p> | ||
* <p> | ||
* This rule should be used in conjunction with the @{@link Flaky} annotation. When this Rule is applied to a test | ||
* class, any test method with this annotation will be invoked up to 3 times or until it succeeds. | ||
* </p> | ||
* <p> | ||
* Tests should <em>not</em> be marked @{@link Flaky} for a long period of time. Every usage should be | ||
* accompanied by a GitHub issue URL, and should be subject to review at a suitable point in the (near) future. | ||
* Should the review date pass without the test's instability being fixed, the retry behaviour will cease to have an | ||
* effect and the test will be allowed to sporadically fail again. | ||
* </p> | ||
*/ | ||
@Slf4j | ||
public class FlakyTestJUnit4RetryRule implements TestRule { | ||
|
||
@Override | ||
public Statement apply(Statement base, Description description) { | ||
|
||
final Flaky annotation = description.getAnnotation(Flaky.class); | ||
|
||
rnorth marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if (annotation != null) { | ||
if (annotation.githubIssueUrl().trim().length() == 0) { | ||
throw new IllegalArgumentException("A GitHub issue URL must be set for usages of the @Flaky annotation"); | ||
} | ||
|
||
final int maxTries = annotation.maxTries(); | ||
rnorth marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
final LocalDate reviewDate; | ||
try { | ||
reviewDate = LocalDate.parse(annotation.reviewDate()); | ||
} catch (DateTimeParseException e) { | ||
throw new IllegalArgumentException("@Flaky reviewDate could not be parsed. Please provide a date in yyyy-mm-dd format"); | ||
} | ||
|
||
// the annotation should only have an effect before the review date, to encourage review and resolution | ||
if ( LocalDate.now().isBefore(reviewDate) ) { | ||
return new RetryingStatement(base, description, maxTries); | ||
} | ||
} | ||
|
||
// otherwise leave the statement as-is | ||
return base; | ||
} | ||
|
||
private static class RetryingStatement extends Statement { | ||
private final Statement base; | ||
private final Description description; | ||
private final int maxTries; | ||
|
||
RetryingStatement(Statement base, Description description, int maxTries) { | ||
this.base = base; | ||
this.description = description; | ||
this.maxTries = maxTries; | ||
} | ||
|
||
@Override | ||
public void evaluate() { | ||
|
||
int attempts = 0; | ||
final List<Throwable> causes = new ArrayList<>(); | ||
|
||
while (++attempts <= maxTries) { | ||
try { | ||
base.evaluate(); | ||
return; | ||
} catch (Throwable throwable) { | ||
log.warn("Retrying @Flaky-annotated test: {}", description.getDisplayName()); | ||
causes.add(throwable); | ||
} | ||
} | ||
|
||
throw new IllegalStateException( | ||
"@Flaky-annotated test failed despite retries.", | ||
new MultipleFailureException(causes)); | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is a way to disable publication of these modules. Not sure if it's the best.
I've tested this manually using
./gradlew -x test publishToMavenLocal
and verified that these three modules are no longer created under~/.m2/repository/org/testcontainers
We should obviously be cautious to check the same applies to the bintray publication as well.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
AFAIK
artifacts.removeAll()
will do the trick (no publishing if there are no artifacts to publish)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I honestly can't figure out how to make this work - my Gradle fu is clearly not strong enough. Unless there's something wrong with my approach I'd prefer to keep with something that I can understand!