Discrepancies with dependsOnMethods when dependencies span across classes

[krmahadevan]

TestNG Version

Note: only the latest version is supported
7.6.1

Expected behavior

dependsOnMethods attribute should honour upstream failures when it spans across instances rather than the same instance.

Actual behavior

Upstream failures are confined to ONLY the current instance but does not span across instances.

Is the issue reproducible on runner?

• Shell
• Maven
• Gradle
• Ant
• Eclipse
• IntelliJ
• NetBeans

Issue summary

Let's say we have two test classes.

• ClassA

  • aa()

• ClassB

  • ba()

Lets assume that aa() has been defined as below:

@Test(dependsOnMethods = "ClassB.ba")
public void aa() {}

Now if both ClassA and ClassB have been chosen for execution, then TestNG is currently doing the following:

• ba() is first executed before running aa(). This assures that TestNG is honouring cross class method dependencies.

But if ba() fails, then TestNG should be skipping ClassA.aa() (because of the dependency) but it does not do that and instead runs aa() despite there being a failure in upstream.

The same behaviour is seen even when we use dependsOnMethods to refer to a method in a inner nested class.

Test case sample

Please, share the test case (as small as possible) which shows the issue

ClassATestClassSample

package com.rationaleemotions.dependencies;

import org.testng.annotations.Test;

public class ClassATestClassSample {

  @Test(dependsOnMethods = "com.rationaleemotions.dependencies.ClassBTestClassSample.passingMethodB")
  public void methodA() {}

  @Test(dependsOnMethods = "com.rationaleemotions.dependencies.ClassBTestClassSample.failingMethodB")
  public void methodB() {
  }
}

ClassBTestClassSample

package com.rationaleemotions.dependencies;

import org.testng.Assert;
import org.testng.annotations.Test;

public class ClassBTestClassSample {

  @Test
  public void passingMethodB() {}

  @Test
  public void failingMethodB() {
    Assert.fail();
  }
}

NestedTestClassSample

package com.rationaleemotions.dependencies;

import org.testng.Assert;
import org.testng.annotations.Test;

public class NestedTestClassSample {

  @Test(dependsOnMethods =
      "com.rationaleemotions.dependencies.NestedTestClassSample$InnerClassSample.failingMethodB")
  public void methodA() {}

  public static class InnerClassSample {

    @Test
    public void failingMethodB() {
      Assert.fail();
    }
  }
}

TestResultsGatherer

package com.rationaleemotions.dependencies;

import java.util.ArrayList;
import java.util.List;
import org.testng.ITestListener;
import org.testng.ITestResult;

public class TestResultsGatherer implements ITestListener {

  private final List<String> passed = new ArrayList<>();
  private final List<String> failed = new ArrayList<>();
  private final List<String> skipped = new ArrayList<>();

  public List<String> getFailed() {
    return failed;
  }

  public List<String> getPassed() {
    return passed;
  }

  public List<String> getSkipped() {
    return skipped;
  }

  @Override
  public void onTestSuccess(ITestResult result) {
    passed.add(result.getMethod().getQualifiedName());
  }

  @Override
  public void onTestFailure(ITestResult result) {
    failed.add(result.getMethod().getQualifiedName());
  }

  @Override
  public void onTestSkipped(ITestResult result) {
    skipped.add(result.getMethod().getQualifiedName());
  }
}

Test case

import static org.assertj.core.api.Assertions.assertThat;

import org.testng.TestNG;
import org.testng.annotations.Test;

public class IssueTest {

  @Test
  public void runTestWithNestedClasses() {
    TestNG testng = new TestNG();
    testng.setTestClasses(new Class[]{NestedTestClassSample.class});
    TestResultsGatherer gatherer = new TestResultsGatherer();
    testng.addListener(gatherer);
    testng.setVerbose(2);
    testng.run();
    String outer = NestedTestClassSample.class.getCanonicalName();
    String inner = outer + "$" + NestedTestClassSample.InnerClassSample.class.getSimpleName();
    assertThat(gatherer.getFailed())
        .containsExactly(inner + ".failingMethodB");
    assertThat(gatherer.getPassed())
        .isEmpty();
    assertThat(gatherer.getSkipped())
        .containsExactly(outer + ".methodA");
  }

  @Test
  public void runTest() {
    TestNG testng = new TestNG();
    testng.setTestClasses(new Class[]{
        ClassATestClassSample.class, ClassBTestClassSample.class
    });
    TestResultsGatherer gatherer = new TestResultsGatherer();
    testng.addListener(gatherer);
    testng.setVerbose(2);
    testng.run();
    String classB = ClassBTestClassSample.class.getCanonicalName();
    String classA = ClassATestClassSample.class.getCanonicalName();
    assertThat(gatherer.getFailed())
        .containsExactly(classB + ".failingMethodB");
    assertThat(gatherer.getPassed())
        .containsExactly(classA + ".methodA", classB + ".passingMethodB");
    assertThat(gatherer.getSkipped())
        .containsExactly(classA + ".methodB");
  }
}