diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/IncludesExcludesTestInheritanceIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/IncludesExcludesTestInheritanceIT.java new file mode 100644 index 0000000000..2808afbd89 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/IncludesExcludesTestInheritanceIT.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.maven.surefire.its; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.maven.shared.verifier.VerificationException; +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.apache.maven.surefire.its.fixture.SurefireLauncher; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; + +@RunWith(Parameterized.class) +public class IncludesExcludesTestInheritanceIT extends SurefireJUnit4IntegrationTestCase { + + @Parameter + @SuppressWarnings("checkstyle:visibilitymodifier") + public String provider; + + @Parameters(name = "Provider: {0}") + public static List providers() { + return Arrays.asList("JUnit", "TestNG"); + } + + private final String[] allTests = new String[] { + "OtherClass#testIncluded", + "OtherClass#testNotIncluded", + "SubClass#testInSubClass", + "SuperClass#testInSuperClass" + }; + + private final String[] subClassTests = new String[] {"SubClass#testInSubClass", "SuperClass#testInSuperClass"}; + + @Test + public void testInclusionOfInheritedTest() throws VerificationException { + // includesFile: + // SubClass + verifyOnlyExpectedTestsExecuted(unpack().activateProfile("profile1"), subClassTests); + + // same with -Dtest option + verifyOnlyExpectedTestsExecuted(unpack().addGoal("-Dtest=SubClass"), subClassTests); + } + + @Test + public void testInclusionOfInheritedTestWhenMethodFilterUsed() throws VerificationException { + // includesFile: + // SubClass + // OtherClass#testIncluded + verifyOnlyExpectedTestsExecuted( + unpack().activateProfile("profile2"), + "OtherClass#testIncluded", + "SubClass#testInSubClass", + "SuperClass#testInSuperClass"); + } + + @Test + public void testInclusionOfInheritedTestOnly() throws VerificationException { + // includesFile: + // SubClass#testInSuperClass + verifyOnlyExpectedTestsExecuted(unpack().activateProfile("profile3"), "SuperClass#testInSuperClass"); + + // same with -Dtest option + verifyOnlyExpectedTestsExecuted( + unpack().addGoal("-Dtest=SubClass#testInSuperClass"), "SuperClass#testInSuperClass"); + } + + @Test + public void testExclusionOfInheritedTest() throws VerificationException { + // includesFile: + // SubClass + // excludesFile: + // SubClass#testInSuperClass + verifyOnlyExpectedTestsExecuted(unpack().activateProfile("profile4"), "SubClass#testInSubClass"); + } + + @Test + public void testIncorrectExclusionOfInheritedTest() throws VerificationException { + // includesFile: + // SubClass + // excludesFile: + // SuperClass + verifyOnlyExpectedTestsExecuted(unpack().activateProfile("profile5"), subClassTests); + + // same with and + verifyOnlyExpectedTestsExecuted(unpack().activateProfile("profile6"), subClassTests); + } + + @Test + public void testIncorrectExclusionOfInheritedTestWithMethodFilter() throws VerificationException { + // includesFile: + // SubClass + // excludesFile: + // SuperClass#testInSuperClass + verifyOnlyExpectedTestsExecuted(unpack().activateProfile("profile7"), subClassTests); + } + + private void verifyOnlyExpectedTestsExecuted(SurefireLauncher launcher, String... expectedTests) + throws VerificationException { + verifyOnlyExpectedTestsExecuted(launcher, new HashSet<>(Arrays.asList(expectedTests))); + } + + private void verifyOnlyExpectedTestsExecuted(SurefireLauncher launcher, Set expectedTests) + throws VerificationException { + String module = provider.toLowerCase(); + OutputValidator outputValidator = launcher.addGoal("-pl=" + module).executeTest(); + for (String test : allTests) { + if (expectedTests.contains(test)) { + assertThatTestWasExecuted(outputValidator, test); + } else { + assertThatTestWasNotExecuted(outputValidator, test); + } + } + launcher.getSubProjectValidator(module).verifyErrorFree(expectedTests.size()); + } + + private void assertThatTestWasExecuted(OutputValidator outputValidator, String test) throws VerificationException { + assertEquals( + String.format("Test %s was not executed with %s provider.", test, provider), + 1, + testExecutionLogOccurrences(outputValidator, test)); + } + + private void assertThatTestWasNotExecuted(OutputValidator outputValidator, String test) + throws VerificationException { + assertEquals( + String.format("Test %s was executed with %s provider.", test, provider), + 0, + testExecutionLogOccurrences(outputValidator, test)); + } + + private int testExecutionLogOccurrences(OutputValidator outputValidator, String test) throws VerificationException { + return outputValidator + .loadLogLines(containsString(testExecutionLog(test))) + .size(); + } + + private String testExecutionLog(String test) { + return String.format("%s: %s executed", provider, test); + } + + private SurefireLauncher unpack() { + return unpack("/includes-excludes-test-inheritance"); + } +} diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions4.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions4.txt new file mode 100644 index 0000000000..65451ac487 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions4.txt @@ -0,0 +1 @@ +SubClass#testInSuperClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions5.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions5.txt new file mode 100644 index 0000000000..1d3feb2926 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions5.txt @@ -0,0 +1 @@ +SuperClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions7.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions7.txt new file mode 100644 index 0000000000..c601de60c8 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/exclusions7.txt @@ -0,0 +1 @@ +SuperClass#testInSuperClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions1.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions1.txt new file mode 100644 index 0000000000..7dfc85a8f9 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions1.txt @@ -0,0 +1 @@ +SubClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions2.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions2.txt new file mode 100644 index 0000000000..1e561a3309 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions2.txt @@ -0,0 +1,2 @@ +SubClass +OtherClass#testIncluded \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions3.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions3.txt new file mode 100644 index 0000000000..65451ac487 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions3.txt @@ -0,0 +1 @@ +SubClass#testInSuperClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions4.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions4.txt new file mode 100644 index 0000000000..7dfc85a8f9 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions4.txt @@ -0,0 +1 @@ +SubClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions5.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions5.txt new file mode 100644 index 0000000000..7dfc85a8f9 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions5.txt @@ -0,0 +1 @@ +SubClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions7.txt b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions7.txt new file mode 100644 index 0000000000..7dfc85a8f9 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/inclusions7.txt @@ -0,0 +1 @@ +SubClass \ No newline at end of file diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/pom.xml b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/pom.xml new file mode 100644 index 0000000000..5bc325b6b9 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/pom.xml @@ -0,0 +1,42 @@ + + + + + + 4.0.0 + + + org.apache.maven.plugins.surefire + maven-it-includes-excludes-test-inheritance + 1.0 + + + junit + 1.0 + + + + org.junit.jupiter + junit-jupiter + 5.14.1 + test + + + diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/OtherClass.java b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/OtherClass.java new file mode 100644 index 0000000000..8ce62ebb3f --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/OtherClass.java @@ -0,0 +1,17 @@ +package junit; + +import org.junit.jupiter.api.Test; + +public class OtherClass { + + @Test + public void testIncluded() { + System.out.println("JUnit: OtherClass#testIncluded executed"); + } + + @Test + public void testNotIncluded() { + System.out.println("JUnit: OtherClass#testNotIncluded executed"); + } + +} diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/SubClass.java b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/SubClass.java new file mode 100644 index 0000000000..e2eeedc37c --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/SubClass.java @@ -0,0 +1,12 @@ +package junit; + +import org.junit.jupiter.api.Test; + +public class SubClass extends SuperClass { + + @Test + public void testInSubClass() { + System.out.println("JUnit: SubClass#testInSubClass executed"); + } + +} diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/SuperClass.java b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/SuperClass.java new file mode 100644 index 0000000000..2e7a9762bf --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/junit/src/test/java/SuperClass.java @@ -0,0 +1,12 @@ +package junit; + +import org.junit.jupiter.api.Test; + +public abstract class SuperClass { + + @Test + public void testInSuperClass() { + System.out.println("JUnit: SuperClass#testInSuperClass executed"); + } + +} diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/pom.xml b/surefire-its/src/test/resources/includes-excludes-test-inheritance/pom.xml new file mode 100644 index 0000000000..804595a7a3 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/pom.xml @@ -0,0 +1,146 @@ + + + + + + 4.0.0 + + org.apache.maven.plugins.surefire + maven-it-includes-excludes-test-inheritance + 1.0 + pom + + + 1.8 + 1.8 + + + + testng + junit + + + + + + maven-surefire-plugin + ${surefire.version} + + + + + + + profile1 + + + + maven-surefire-plugin + + ${project.parent.basedir}/inclusions1.txt + + + + + + + profile2 + + + + maven-surefire-plugin + + ${project.parent.basedir}/inclusions2.txt + + + + + + + profile3 + + + + maven-surefire-plugin + + ${project.parent.basedir}/inclusions3.txt + + + + + + + profile4 + + + + maven-surefire-plugin + + ${project.parent.basedir}/inclusions4.txt + ${project.parent.basedir}/exclusions4.txt + + + + + + + profile5 + + + + maven-surefire-plugin + + ${project.parent.basedir}/inclusions5.txt + ${project.parent.basedir}/exclusions5.txt + + + + + + + profile6 + + + + maven-surefire-plugin + + SubClass + SuperClass + + + + + + + profile7 + + + + maven-surefire-plugin + + ${project.parent.basedir}/inclusions7.txt + ${project.parent.basedir}/exclusions7.txt + + + + + + + diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/pom.xml b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/pom.xml new file mode 100644 index 0000000000..26d88a8dea --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/pom.xml @@ -0,0 +1,42 @@ + + + + + + 4.0.0 + + + org.apache.maven.plugins.surefire + maven-it-includes-excludes-test-inheritance + 1.0 + + + testng + 1.0 + + + + org.testng + testng + 7.5.1 + test + + + diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/OtherClass.java b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/OtherClass.java new file mode 100644 index 0000000000..16a49774fe --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/OtherClass.java @@ -0,0 +1,17 @@ +package testng; + +import org.testng.annotations.Test; + +public class OtherClass { + + @Test + public void testIncluded() { + System.out.println("TestNG: OtherClass#testIncluded executed"); + } + + @Test + public void testNotIncluded() { + System.out.println("TestNG: OtherClass#testNotIncluded executed"); + } + +} diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/SubClass.java b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/SubClass.java new file mode 100644 index 0000000000..9618e828f3 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/SubClass.java @@ -0,0 +1,12 @@ +package testng; + +import org.testng.annotations.Test; + +public class SubClass extends SuperClass { + + @Test + public void testInSubClass() { + System.out.println("TestNG: SubClass#testInSubClass executed"); + } + +} diff --git a/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/SuperClass.java b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/SuperClass.java new file mode 100644 index 0000000000..8b526134d3 --- /dev/null +++ b/surefire-its/src/test/resources/includes-excludes-test-inheritance/testng/src/test/java/SuperClass.java @@ -0,0 +1,12 @@ +package testng; + +import org.testng.annotations.Test; + +public abstract class SuperClass { + + @Test + public void testInSuperClass() { + System.out.println("TestNG: SuperClass#testInSuperClass executed"); + } + +} diff --git a/surefire-providers/surefire-testng-utils/src/main/java/org/apache/maven/surefire/testng/utils/MethodSelector.java b/surefire-providers/surefire-testng-utils/src/main/java/org/apache/maven/surefire/testng/utils/MethodSelector.java index b5930ad4ef..abc704558d 100644 --- a/surefire-providers/surefire-testng-utils/src/main/java/org/apache/maven/surefire/testng/utils/MethodSelector.java +++ b/surefire-providers/surefire-testng-utils/src/main/java/org/apache/maven/surefire/testng/utils/MethodSelector.java @@ -61,7 +61,9 @@ private static boolean shouldRun(ITestNGMethod test) { boolean hasTestResolver = resolver != null && !resolver.isEmpty(); if (hasTestResolver) { boolean run = false; - for (Class clazz = test.getRealClass(); + Class testClass = + test.getTestClass() != null ? test.getTestClass().getRealClass() : test.getRealClass(); + for (Class clazz = testClass; !run && clazz != null && clazz != Object.class; clazz = clazz.getSuperclass()) { run = resolver.shouldRun(clazz, test.getMethodName()); diff --git a/surefire-providers/surefire-testng-utils/src/test/java/testng/utils/MethodSelectorTest.java b/surefire-providers/surefire-testng-utils/src/test/java/testng/utils/MethodSelectorTest.java index c0e7838bda..93e0cfd430 100644 --- a/surefire-providers/surefire-testng-utils/src/test/java/testng/utils/MethodSelectorTest.java +++ b/surefire-providers/surefire-testng-utils/src/test/java/testng/utils/MethodSelectorTest.java @@ -44,6 +44,18 @@ public void testInclusionOfMethodFromBaseClass() { assertTrue(include); } + public void testInclusionOfInheritedMethod() { + MethodSelector selector = new MethodSelector(); + DefaultMethodSelectorContext context = new DefaultMethodSelectorContext(); + ITestNGMethod testngMethod = new FakeTestNGMethod( + BaseClassSample.class, "baseClassMethodToBeIncluded", new FakeTestClass(ChildClassSample.class)); + TestListResolver resolver = + new TestListResolver(ChildClassSample.class.getName() + "#baseClassMethodToBeIncluded"); + MethodSelector.setTestListResolver(resolver); + boolean include = selector.includeMethod(context, testngMethod, true); + assertTrue(include); + } + public void testNoInclusionOfMethodFromBaseClass() { MethodSelector selector = new MethodSelector(); DefaultMethodSelectorContext context = new DefaultMethodSelectorContext(); @@ -67,10 +79,16 @@ public void testInclusionOfMethodFromSubClass() { private static class FakeTestNGMethod implements ITestNGMethod { private final Class clazz; private final String methodName; + private final ITestClass testClass; FakeTestNGMethod(Class clazz, String methodName) { + this(clazz, methodName, new FakeTestClass(clazz)); + } + + FakeTestNGMethod(Class clazz, String methodName, ITestClass testClass) { this.clazz = clazz; this.methodName = methodName; + this.testClass = testClass; } @Override @@ -80,7 +98,7 @@ public Class getRealClass() { @Override public ITestClass getTestClass() { - return null; + return testClass; } @Override @@ -313,4 +331,101 @@ public int compareTo(Object o) { return 0; } } + + private static class FakeTestClass implements ITestClass { + + private final Class realClass; + + FakeTestClass(Class realClass) { + this.realClass = realClass; + } + + @Override + public String getName() { + return ""; + } + + @Override + public String getTestName() { + return ""; + } + + @Override + public Class getRealClass() { + return realClass; + } + + @Override + public Object[] getInstances(boolean reuse) { + return new Object[0]; + } + + @Override + public long[] getInstanceHashCodes() { + return new long[0]; + } + + @Override + public void addInstance(Object instance) {} + + @Override + public int getInstanceCount() { + return 0; + } + + @Override + public ITestNGMethod[] getTestMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getBeforeTestMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getAfterTestMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getBeforeClassMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getAfterClassMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getBeforeSuiteMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getAfterSuiteMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getBeforeTestConfigurationMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getAfterTestConfigurationMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getBeforeGroupsMethods() { + return new ITestNGMethod[0]; + } + + @Override + public ITestNGMethod[] getAfterGroupsMethods() { + return new ITestNGMethod[0]; + } + } }