Skip to content

Commit

Permalink
Merge pull request #1487 from selenareneephillips/add_logging_improve…
Browse files Browse the repository at this point in the history 
…_error_messages_parallelization_tests

Added logging, comments and improved error messaging + fixed bug in parallelization test suite
  • Loading branch information
@cbeust
cbeust committed Jul 24, 2017
2 parents 971aca1 + 929a377 commit b47fc4e
Show file tree
Hide file tree
Showing 15 changed files with 826 additions and 82 deletions.
379 changes: 312 additions & 67 deletions src/test/java/test/thread/parallelization/BaseParallelizationTest.java
View file

Large diffs are not rendered by default.

19 changes: 17 additions & 2 deletions src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
package test.thread.parallelization;

import com.google.common.collect.Multimap;
import org.testng.ITestNGListener;
import org.testng.TestNG;

import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import org.testng.xml.XmlSuite;

import test.thread.parallelization.TestNgRunStateTracker.EventLog;
import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample;

import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand Down Expand Up @@ -46,6 +48,12 @@
*/
public class ParallelByMethodsTestCase1Scenario1 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario1.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE = "SingleTestSuite";
private static final String TEST = "SingleTestClassTest";

Expand Down Expand Up @@ -77,6 +85,13 @@ public void setUp() {
TestNG tng = create(suite);
tng.addListener((ITestNGListener)new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario1. This test scenario consists of a " +
"single suite with a single test which consists of one test class with five test methods. There " +
"are no dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE,TEST,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName(), 5});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
49 changes: 49 additions & 0 deletions src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import test.thread.parallelization.TestNgRunStateTracker.EventLog;

Expand Down Expand Up @@ -77,6 +79,13 @@
* 10) There are no method exclusions
*/
public class ParallelByMethodsTestCase1Scenario2 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario2.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE_A = "TestSuiteA";
private static final String SUITE_B = "TestSuiteB";
private static final String SUITE_C = "TestSuiteC";
Expand Down Expand Up @@ -208,6 +217,46 @@ public void setUp() {
TestNG tng = create(suiteOne, suiteTwo, suiteThree);
tng.addListener((ITestNGListener) new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario2. This test scenario consists of three " +
"sequentially executed suites with 1, 2 and 3 tests respectively. One test for a suite consists of a " +
"single test class while the rest shall consist of more than one test class. There are no " +
"dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() +
", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_B,
TestClassDThreeMethodsWithNoDepsSample.class + ", " +
TestClassBFourMethodsWithNoDepsSample.class + ", " +
TestClassFSixMethodsWithNoDepsSample.class,
20});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_A,
TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassIFiveMethodsWithNoDepsSample.class,
10});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_B,
TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassKFiveMethodsWithNoDepsSample.class,
5});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_C,
TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(),
12});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
30 changes: 29 additions & 1 deletion src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

import org.testng.ITestNGListener;
import org.testng.TestNG;

import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import org.testng.xml.XmlSuite;

import test.thread.parallelization.TestNgRunStateTracker.EventLog;
Expand All @@ -19,6 +21,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand Down Expand Up @@ -62,6 +66,12 @@
*/
public class ParallelByMethodsTestCase2Scenario1 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario1.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE_A = "TestSuiteA";
private static final String SUITE_B = "TestSuiteB";

Expand Down Expand Up @@ -125,7 +135,6 @@ public void setUp() {
suiteTwo.setParallel(XmlSuite.ParallelMode.METHODS);
suiteTwo.setThreadCount(14);


addParams(suiteOne, SUITE_A, SUITE_A_TEST_A, "100");

addParams(suiteTwo, SUITE_B, SUITE_B_TEST_A, "100");
Expand All @@ -135,6 +144,25 @@ public void setUp() {
tng.setSuiteThreadPoolSize(2);
tng.addListener((ITestNGListener) new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario1. This test scenario consists of two " +
"suites with 1 and 2 tests respectively. The suites run in parallel and the thread pool size is 2. " +
"One test for a suite shall consist of a single test class while the rest shall consist of more than " +
"one test class. There are no dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() +
", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 14});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_B,
TestClassDThreeMethodsWithNoDepsSample.class + ", " +
TestClassBFourMethodsWithNoDepsSample.class + ", " +
TestClassFSixMethodsWithNoDepsSample.class,
14});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
49 changes: 49 additions & 0 deletions src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand Down Expand Up @@ -71,6 +73,13 @@
* 11) There are no method exclusions
*/
public class ParallelByMethodsTestCase2Scenario2 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario2.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE_A = "TestSuiteA";
private static final String SUITE_B = "TestSuiteB";
private static final String SUITE_C = "TestSuiteC";
Expand Down Expand Up @@ -204,6 +213,46 @@ public void setUp() {
tng.setSuiteThreadPoolSize(2);
tng.addListener((ITestNGListener) new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario2. This test scenario consists of three " +
"suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is 3 " +
"One test for a suite shall consist of a single test class while the rest shall consist of more than " +
"one test class. There are no dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() +
", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_B,
TestClassDThreeMethodsWithNoDepsSample.class + ", " +
TestClassBFourMethodsWithNoDepsSample.class + ", " +
TestClassFSixMethodsWithNoDepsSample.class,
20});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_A,
TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassIFiveMethodsWithNoDepsSample.class,
10});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_B,
TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassKFiveMethodsWithNoDepsSample.class,
5});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_C,
TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(),
12});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
51 changes: 50 additions & 1 deletion src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,13 @@

import test.thread.parallelization.TestNgRunStateTracker.EventLog;
import test.thread.parallelization.sample.TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample;
import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand All @@ -25,6 +30,7 @@
import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartEventLog;
import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartThreadId;

import static test.thread.parallelization.TestNgRunStateTracker.getTestMethodLevelEventLogsForTest;
import static test.thread.parallelization.TestNgRunStateTracker.reset;

/**
Expand All @@ -48,6 +54,13 @@
* 7) There are no method exclusions
*/
public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase3Scenario1.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE = "SingleTestSuite";
private static final String TEST = "SingleTestClassTest";

Expand All @@ -64,6 +77,8 @@ public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest

private Long testListenerOnStartThreadId;

private Map<String, Integer> expectedInvocationCounts = new HashMap<>();

@BeforeClass
public void setUp() {
reset();
Expand All @@ -72,6 +87,26 @@ public void setUp() {
suite.setParallel(XmlSuite.ParallelMode.METHODS);
suite.setThreadCount(15);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodA", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodB", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodC", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodD", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodE", 3);

createXmlTest(suite, TEST, TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class);

addParams(suite, SUITE, TEST, "100", "paramOne,paramTwo,paramThree");
Expand All @@ -80,6 +115,14 @@ public void setUp() {

tng.addListener((ITestNGListener)new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase3Scenario1. This test scenario consists of a " +
"single suite with a single test consisting of a single test class with five methods with a data " +
"provider specifying 3 sets of data. There are no dependencies or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE,TEST,
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName(), 15});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down Expand Up @@ -168,7 +211,13 @@ public void verifyThatMethodLevelEventsRunInDifferentThreadsFromSuiteAndTestLeve
//Verifies that the test methods execute in different threads in parallel fashion.
@Test
public void verifyThatTestMethodsRunInParallelThreads() {
verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5);

verifyParallelTestMethodsWithNonParallelDataProvider(
getTestMethodLevelEventLogsForTest(SUITE, TEST), TEST, expectedInvocationCounts,
5, 5
);

//verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5);
}

//Verifies that all the test method level events for any given test method run in the same thread.
Expand Down
Loading

0 comments on commit b47fc4e

Please sign in to comment.