Skip to content

Commit

Permalink
Prohibit usage of JUnit5 class lifecycle
Browse files Browse the repository at this point in the history 
JUnit5 offers to use the same class instance over several methods.
This however is not in line with the scenario concept of JGiven. The JGivenExtension removes the scenario after the exection of each state, resulting in failure if the next scenario is executed on the same instance

Moreover, it is unclear how scenarios that share the same instance react when these methods are executed in parallel.

In addition, the JUnit 5 testframework tests were adapted. It turns out that when JUnit5 fails on the class level it returns a successful report model for method but a failure for the class level.

Signed-off-by: l-1sqared <[email protected]>
  • Loading branch information
@l-1squared
l-1squared committed Feb 10, 2022
1 parent 083d7fb commit 1bc9d4a
Show file tree
Hide file tree
Showing 8 changed files with 191 additions and 105 deletions.
1 change: 1 addition & 0 deletions docs/junit5.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public class JGiven5ScenarioTest
}
}
----
Please note that JGiven is built around each scenario, i.e. test method, having its own test instance. Therefore, we cannot support JUnit's per-class test instance lifecycle. An exception will be thrown if any such attempt is made.

=== Example Project

Expand Down
43 changes: 32 additions & 11 deletions jgiven-junit5/src/main/java/com/tngtech/jgiven/junit5/JGivenExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,25 @@
import static com.tngtech.jgiven.report.model.ExecutionStatus.FAILED;
import static com.tngtech.jgiven.report.model.ExecutionStatus.SUCCESS;

import java.util.EnumSet;

import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.extension.*;
import org.junit.jupiter.api.extension.ExtensionContext.Namespace;

import com.tngtech.jgiven.base.ScenarioTestBase;
import com.tngtech.jgiven.config.AbstractJGivenConfiguration;
import com.tngtech.jgiven.config.ConfigurationUtil;
import com.tngtech.jgiven.exception.JGivenWrongUsageException;
import com.tngtech.jgiven.impl.ScenarioBase;
import com.tngtech.jgiven.impl.ScenarioHolder;
import com.tngtech.jgiven.report.impl.CommonReportHelper;
import com.tngtech.jgiven.report.model.ReportModel;
import java.util.EnumSet;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.AfterTestExecutionCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ExtensionContext.Namespace;
import org.junit.jupiter.api.extension.TestInstancePostProcessor;

/**
* This extension enables JGiven for JUnit 5 Tests.
Expand Down Expand Up @@ -47,6 +52,8 @@ public class JGivenExtension implements

@Override
public void beforeAll(ExtensionContext context) {
validatePerMethodLifecycle(context);

ReportModel reportModel = new ReportModel();
reportModel.setTestClass(context.getTestClass().get());
if (!context.getDisplayName().equals(context.getTestClass().get().getSimpleName())) {
Expand All @@ -62,6 +69,7 @@ public void beforeAll(ExtensionContext context) {
}
}


@Override
public void afterAll(ExtensionContext context) {
ScenarioHolder.get().removeScenarioOfCurrentThread();
Expand Down Expand Up @@ -96,10 +104,6 @@ public void afterTestExecution(ExtensionContext context) throws Exception {
}
}

private ScenarioBase getScenario() {
return ScenarioHolder.get().getScenarioOfCurrentThread();
}

@Override
public void postProcessTestInstance(Object testInstance, ExtensionContext context) {
ScenarioBase currentScenario = ScenarioHolder.get().getScenarioOfCurrentThread();
Expand All @@ -125,4 +129,21 @@ public void postProcessTestInstance(Object testInstance, ExtensionContext contex
scenario.getExecutor().readScenarioState(testInstance);
}

private void validatePerMethodLifecycle(ExtensionContext context) {
if (isPerClassLifecycle(context)) {
throw new JGivenWrongUsageException(
"JGiven does not support keeping a test instance over multiple scenarios. Please use Lifecycle '"
+ TestInstance.Lifecycle.PER_METHOD + "'.");
}
}

private boolean isPerClassLifecycle(ExtensionContext context) {
return context.getTestInstanceLifecycle()
.filter(lifecycle -> TestInstance.Lifecycle.PER_CLASS == lifecycle)
.isPresent();
}

private ScenarioBase getScenario() {
return ScenarioHolder.get().getScenarioOfCurrentThread();
}
}
150 changes: 77 additions & 73 deletions jgiven-tests/src/main/java/com/tngtech/jgiven/tests/TestScenarioRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

public class TestScenarioRepository {


public static class SearchCriteria {
public boolean pending = false;
public boolean failing = false;
Expand All @@ -21,53 +22,53 @@ public static class SearchCriteria {
public Integer numberOfParameters;
public String testClassDescription;

public boolean matches( ScenarioCriteria criteria ) {
if( pending != criteria.pending) {
public boolean matches(ScenarioCriteria criteria) {
if (pending != criteria.pending) {
return false;
}

if( failIfPassed != null && !failIfPassed.equals( criteria.failIfPassed ) ) {
if (failIfPassed != null && !failIfPassed.equals(criteria.failIfPassed)) {
return false;
}

if( executeSteps != null && !executeSteps.equals( criteria.executeSteps ) ) {
if (executeSteps != null && !executeSteps.equals(criteria.executeSteps)) {
return false;
}

if( failing != criteria.failing ) {
if (failing != criteria.failing) {
return false;
}

if( numberOfSteps != null && !numberOfSteps.equals( criteria.numberOfSteps ) ) {
if (numberOfSteps != null && !numberOfSteps.equals(criteria.numberOfSteps)) {
return false;
}

if( numberOfFailingStages != null && !numberOfFailingStages.equals( criteria.numberOfFailingStages ) ) {
if (numberOfFailingStages != null && !numberOfFailingStages.equals(criteria.numberOfFailingStages)) {
return false;
}

if( failingStep != null && !failingStep.equals( criteria.failingStep ) ) {
if (failingStep != null && !failingStep.equals(criteria.failingStep)) {
return false;
}

if( stageWithFailingAfterStageMethod != null
&& !stageWithFailingAfterStageMethod.equals( criteria.stageWithFailingAfterStageMethod ) ) {
if (stageWithFailingAfterStageMethod != null
&& !stageWithFailingAfterStageMethod.equals(criteria.stageWithFailingAfterStageMethod)) {
return false;
}

if( tagAnnotation != null && !tagAnnotation.equals( criteria.tagAnnotation ) ) {
if (tagAnnotation != null && !tagAnnotation.equals(criteria.tagAnnotation)) {
return false;
}

if( parameterizedRunner != null && !parameterizedRunner.equals( criteria.parameterizedRunner ) ) {
if (parameterizedRunner != null && !parameterizedRunner.equals(criteria.parameterizedRunner)) {
return false;
}

if( numberOfParameters != null && !numberOfParameters.equals(criteria.numberOfParameters) ) {
if (numberOfParameters != null && !numberOfParameters.equals(criteria.numberOfParameters)) {
return false;
}

if( testClassDescription != null && !testClassDescription.equals( criteria.testClassDescription ) ) {
if (testClassDescription != null && !testClassDescription.equals(criteria.testClassDescription)) {
return false;
}

Expand Down Expand Up @@ -109,13 +110,13 @@ public ScenarioCriteria failing() {
return this;
}

public ScenarioCriteria failingStep( int i ) {
public ScenarioCriteria failingStep(int i) {
failing();
failingStep = i;
return this;
}

public ScenarioCriteria numberOfSteps( int n ) {
public ScenarioCriteria numberOfSteps(int n) {
numberOfSteps = n;
return this;
}
Expand All @@ -125,17 +126,17 @@ public ScenarioCriteria tagAnnotation() {
return this;
}

public ScenarioCriteria numberOfFailingStages( int i ) {
public ScenarioCriteria numberOfFailingStages(int i) {
numberOfFailingStages = i;
return this;
}

public ScenarioCriteria stageWithFailingAfterStageMethod( Integer stageWithFailingAfterStageMethod ) {
public ScenarioCriteria stageWithFailingAfterStageMethod(Integer stageWithFailingAfterStageMethod) {
this.stageWithFailingAfterStageMethod = stageWithFailingAfterStageMethod;
return this;
}

public ScenarioCriteria numberOfParameters( int n ) {
public ScenarioCriteria numberOfParameters(int n) {
this.numberOfParameters = n;
return this;
}
Expand All @@ -145,7 +146,7 @@ public ScenarioCriteria parameterizedRunner() {
return this;
}

public ScenarioCriteria testClassDescription( String value ) {
public ScenarioCriteria testClassDescription(String value) {
this.testClassDescription = value;
return this;
}
Expand All @@ -156,130 +157,133 @@ public static class TestScenario {
public String testMethod;
public ScenarioCriteria criteria = new ScenarioCriteria();

public TestScenario( Class<?> testClass ) {
public TestScenario(Class<?> testClass) {
this.testClass = testClass;
}

public TestScenario( String testMethod ) {
public TestScenario(String testMethod) {
this.testMethod = testMethod;
this.testClass = TestScenarios.class;
}

public TestScenario( Class<?> testClass, String testMethod ) {
public TestScenario(Class<?> testClass, String testMethod) {
this.testClass = testClass;
this.testMethod = testMethod;
}
}

final static List<TestScenario> testScenarios = setupTestScenarios();

public static TestScenario findScenario( SearchCriteria searchCriteria ) {
for( TestScenario scenario : testScenarios ) {
if( searchCriteria.matches( scenario.criteria ) ) {
public static TestScenario findScenario(SearchCriteria searchCriteria) {
for (TestScenario scenario : testScenarios) {
if (searchCriteria.matches(scenario.criteria)) {
return scenario;
}
}
throw new IllegalArgumentException( "No matching scenario found" );
throw new IllegalArgumentException("No matching scenario found");
}

private static ScenarioCriteria addTestScenario( List<TestScenario> list, Class<?> testClass ) {
TestScenario testScenario = new TestScenario( testClass );
list.add( testScenario );
private static ScenarioCriteria addTestScenario(List<TestScenario> list, Class<?> testClass) {
TestScenario testScenario = new TestScenario(testClass);
list.add(testScenario);
return testScenario.criteria;
}

private static ScenarioCriteria addTestScenario( List<TestScenario> list, String testMethod ) {
TestScenario testScenario = new TestScenario( testMethod );
list.add( testScenario );
private static ScenarioCriteria addTestScenario(List<TestScenario> list, String testMethod) {
TestScenario testScenario = new TestScenario(testMethod);
list.add(testScenario);
return testScenario.criteria;
}

private static ScenarioCriteria addTestScenario( List<TestScenario> list, Class<?> testClass, String testMethod ) {
TestScenario testScenario = new TestScenario( testClass );
private static ScenarioCriteria addTestScenario(List<TestScenario> list, Class<?> testClass, String testMethod) {
TestScenario testScenario = new TestScenario(testClass);
testScenario.testMethod = testMethod;
list.add( testScenario );
list.add(testScenario);
return testScenario.criteria;
}

private static List<TestScenario> setupTestScenarios() {
List<TestScenario> result = Lists.newArrayList();

addTestScenario( result, "failing_test_with_two_steps" )
.numberOfSteps( 2 )
.failingStep( 1 );
addTestScenario(result, "failing_test_with_two_steps")
.numberOfSteps(2)
.failingStep(1);

addTestScenario( result, "failing_test_with_three_steps" )
.numberOfSteps( 3 )
.failingStep( 1 );
addTestScenario(result, "failing_test_with_three_steps")
.numberOfSteps(3)
.failingStep(1);

addTestScenario( result, "failing_test_with_two_steps_and_second_step_fails" )
.numberOfSteps( 2 )
.failingStep( 2 );
addTestScenario(result, "failing_test_with_two_steps_and_second_step_fails")
.numberOfSteps(2)
.failingStep(2);

addTestScenario( result, "failing_test_with_two_failing_stages" )
.numberOfSteps( 2 )
.numberOfFailingStages( 2 )
.failingStep( 1 );
addTestScenario(result, "failing_test_with_two_failing_stages")
.numberOfSteps(2)
.numberOfFailingStages(2)
.failingStep(1);

addTestScenario( result, "failing_test_where_second_stage_has_a_failing_after_stage_method" )
.numberOfSteps( 2 )
.numberOfFailingStages( 2 )
.stageWithFailingAfterStageMethod( 2 )
.failingStep( 1 );
addTestScenario(result, "failing_test_where_second_stage_has_a_failing_after_stage_method")
.numberOfSteps(2)
.numberOfFailingStages(2)
.stageWithFailingAfterStageMethod(2)
.failingStep(1);

addTestScenario( result, "failing_test_with_Pending_annotation" )
addTestScenario(result, "failing_test_with_Pending_annotation")
.pending()
.numberOfSteps( 2 )
.failingStep( 1 );
.numberOfSteps(2)
.failingStep(1);

addTestScenario( result, "passing_test_with_Pending_annotation" )
addTestScenario(result, "passing_test_with_Pending_annotation")
.pending();

addTestScenario( result, "passing_test_with_Pending_annotation_and_failIfPassed_set_to_true" )
addTestScenario(result, "passing_test_with_Pending_annotation_and_failIfPassed_set_to_true")
.pending()
.failIfPassed();

addTestScenario( result, "failing_test_with_Pending_annotation_and_failIfPassed_set_to_true" )
addTestScenario(result, "failing_test_with_Pending_annotation_and_failIfPassed_set_to_true")
.pending()
.failIfPassed()
.failingStep( 1 );
.failingStep(1);

addTestScenario( result, "failing_test_with_Pending_annotation_and_executeSteps_set_to_true" )
addTestScenario(result, "failing_test_with_Pending_annotation_and_executeSteps_set_to_true")
.pending()
.executeSteps()
.failingStep( 1 );
.failingStep(1);

addTestScenario( result, "test_with_tag_annotation" )
addTestScenario(result, "test_with_tag_annotation")
.tagAnnotation();

addTestScenario( result, TestClassWithParameterizedRunner.class )
addTestScenario(result, TestClassWithParameterizedRunner.class)
.parameterizedRunner()
.numberOfParameters( 2 );
.numberOfParameters(2);

addTestScenario( result, TestClassWithDescription.class, "some_test" )
.testClassDescription( TestClassWithDescription.class.getAnnotation( Description.class ).value() );
addTestScenario(result, TestClassWithDescription.class, "some_test")
.testClassDescription(TestClassWithDescription.class.getAnnotation(Description.class).value());

return result;
}

public static TestScenario testClassWithOnlyIgnoredTests() {
return new TestScenario( TestClassWithOnlyIgnoredTests.class );
return new TestScenario(TestClassWithOnlyIgnoredTests.class);
}

public static TestScenario testClassWithAFailingScenarioAndAFailingAfterStage() {
return new TestScenario( TestWithExceptionsInAfterMethod.class );
return new TestScenario(TestWithExceptionsInAfterMethod.class);
}

public static TestScenario testWithTwoCasesAndTheFirstOneFails() {
return new TestScenario( TestWithTwoCasesAndAFailingOne.class );
return new TestScenario(TestWithTwoCasesAndAFailingOne.class);
}

public static TestScenario junit5TestsWithModificationsInAfterMethod(){
public static TestScenario junit5TestsWithModificationsInAfterMethod() {
return new TestScenario(JUnit5AfterMethodTests.class);
}

public static TestScenario testNgTestWithAFailingCase() {
return new TestScenario( FailingCasesTestNgTest.class );
return new TestScenario(FailingCasesTestNgTest.class);
}

public static TestScenario junit5TestClassWithPerClassLifecycle() {
return new TestScenario(TestWithPerClassLifecycle.class);
}
}
Loading

0 comments on commit 1bc9d4a

Please sign in to comment.