Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added logging, comments and improved error messaging + fixed bug in parallelization test suite #1487

Merged
merged 1 commit into from
Jul 24, 2017
Merged

Added logging, comments and improved error messaging + fixed bug in parallelization test suite #1487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
379 changes: 312 additions & 67 deletions src/test/java/test/thread/parallelization/BaseParallelizationTest.java
View file
Open in desktop

Large diffs are not rendered by default.

19 changes: 17 additions & 2 deletions src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
package test.thread.parallelization;

import com.google.common.collect.Multimap;
import org.testng.ITestNGListener;
import org.testng.TestNG;

import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import org.testng.xml.XmlSuite;

import test.thread.parallelization.TestNgRunStateTracker.EventLog;
import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample;

import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand Down Expand Up @@ -46,6 +48,12 @@
*/
public class ParallelByMethodsTestCase1Scenario1 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario1.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE = "SingleTestSuite";
private static final String TEST = "SingleTestClassTest";

Expand Down Expand Up @@ -77,6 +85,13 @@ public void setUp() {
TestNG tng = create(suite);
tng.addListener((ITestNGListener)new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario1. This test scenario consists of a " +
"single suite with a single test which consists of one test class with five test methods. There " +
"are no dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE,TEST,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName(), 5});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
49 changes: 49 additions & 0 deletions src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import test.thread.parallelization.TestNgRunStateTracker.EventLog;

Expand Down Expand Up @@ -77,6 +79,13 @@
* 10) There are no method exclusions
*/
public class ParallelByMethodsTestCase1Scenario2 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario2.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE_A = "TestSuiteA";
private static final String SUITE_B = "TestSuiteB";
private static final String SUITE_C = "TestSuiteC";
Expand Down Expand Up @@ -208,6 +217,46 @@ public void setUp() {
TestNG tng = create(suiteOne, suiteTwo, suiteThree);
tng.addListener((ITestNGListener) new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario2. This test scenario consists of three " +
"sequentially executed suites with 1, 2 and 3 tests respectively. One test for a suite consists of a " +
"single test class while the rest shall consist of more than one test class. There are no " +
"dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() +
", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_B,
TestClassDThreeMethodsWithNoDepsSample.class + ", " +
TestClassBFourMethodsWithNoDepsSample.class + ", " +
TestClassFSixMethodsWithNoDepsSample.class,
20});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_A,
TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassIFiveMethodsWithNoDepsSample.class,
10});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_B,
TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassKFiveMethodsWithNoDepsSample.class,
5});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_C,
TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(),
12});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
30 changes: 29 additions & 1 deletion src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@

import org.testng.ITestNGListener;
import org.testng.TestNG;

import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import org.testng.xml.XmlSuite;

import test.thread.parallelization.TestNgRunStateTracker.EventLog;
Expand All @@ -19,6 +21,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand Down Expand Up @@ -62,6 +66,12 @@
*/
public class ParallelByMethodsTestCase2Scenario1 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario1.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE_A = "TestSuiteA";
private static final String SUITE_B = "TestSuiteB";

Expand Down Expand Up @@ -125,7 +135,6 @@ public void setUp() {
suiteTwo.setParallel(XmlSuite.ParallelMode.METHODS);
suiteTwo.setThreadCount(14);


addParams(suiteOne, SUITE_A, SUITE_A_TEST_A, "100");

addParams(suiteTwo, SUITE_B, SUITE_B_TEST_A, "100");
Expand All @@ -135,6 +144,25 @@ public void setUp() {
tng.setSuiteThreadPoolSize(2);
tng.addListener((ITestNGListener) new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario1. This test scenario consists of two " +
"suites with 1 and 2 tests respectively. The suites run in parallel and the thread pool size is 2. " +
"One test for a suite shall consist of a single test class while the rest shall consist of more than " +
"one test class. There are no dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() +
", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 14});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_B,
TestClassDThreeMethodsWithNoDepsSample.class + ", " +
TestClassBFourMethodsWithNoDepsSample.class + ", " +
TestClassFSixMethodsWithNoDepsSample.class,
14});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
49 changes: 49 additions & 0 deletions src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand Down Expand Up @@ -71,6 +73,13 @@
* 11) There are no method exclusions
*/
public class ParallelByMethodsTestCase2Scenario2 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario2.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE_A = "TestSuiteA";
private static final String SUITE_B = "TestSuiteB";
private static final String SUITE_C = "TestSuiteC";
Expand Down Expand Up @@ -204,6 +213,46 @@ public void setUp() {
tng.setSuiteThreadPoolSize(2);
tng.addListener((ITestNGListener) new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario2. This test scenario consists of three " +
"suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is 3 " +
"One test for a suite shall consist of a single test class while the rest shall consist of more than " +
"one test class. There are no dependencies, data providers or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() +
", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_B,SUITE_B_TEST_B,
TestClassDThreeMethodsWithNoDepsSample.class + ", " +
TestClassBFourMethodsWithNoDepsSample.class + ", " +
TestClassFSixMethodsWithNoDepsSample.class,
20});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_A,
TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassIFiveMethodsWithNoDepsSample.class,
10});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_B,
TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassKFiveMethodsWithNoDepsSample.class,
5});

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}",
new Object[]{SUITE_C,SUITE_C_TEST_C,
TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " +
TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(),
12});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down
51 changes: 50 additions & 1 deletion src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,13 @@

import test.thread.parallelization.TestNgRunStateTracker.EventLog;
import test.thread.parallelization.sample.TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample;
import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.testng.Assert.assertEquals;

Expand All @@ -25,6 +30,7 @@
import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartEventLog;
import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartThreadId;

import static test.thread.parallelization.TestNgRunStateTracker.getTestMethodLevelEventLogsForTest;
import static test.thread.parallelization.TestNgRunStateTracker.reset;

/**
Expand All @@ -48,6 +54,13 @@
* 7) There are no method exclusions
*/
public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest {

private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase3Scenario1.class.getCanonicalName());

{
logger.setLevel(Level.INFO);
}

private static final String SUITE = "SingleTestSuite";
private static final String TEST = "SingleTestClassTest";

Expand All @@ -64,6 +77,8 @@ public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest

private Long testListenerOnStartThreadId;

private Map<String, Integer> expectedInvocationCounts = new HashMap<>();

@BeforeClass
public void setUp() {
reset();
Expand All @@ -72,6 +87,26 @@ public void setUp() {
suite.setParallel(XmlSuite.ParallelMode.METHODS);
suite.setThreadCount(15);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodA", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodB", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodC", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodD", 3);

expectedInvocationCounts.put(
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() +
".testMethodE", 3);

createXmlTest(suite, TEST, TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class);

addParams(suite, SUITE, TEST, "100", "paramOne,paramTwo,paramThree");
Expand All @@ -80,6 +115,14 @@ public void setUp() {

tng.addListener((ITestNGListener)new TestNgRunStateListener());

logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase3Scenario1. This test scenario consists of a " +
"single suite with a single test consisting of a single test class with five methods with a data " +
"provider specifying 3 sets of data. There are no dependencies or factories.");

logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}",
new Object[]{SUITE,TEST,
TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName(), 15});

tng.run();

suiteLevelEventLogs = getAllSuiteLevelEventLogs();
Expand Down Expand Up @@ -168,7 +211,13 @@ public void verifyThatMethodLevelEventsRunInDifferentThreadsFromSuiteAndTestLeve
//Verifies that the test methods execute in different threads in parallel fashion.
@Test
public void verifyThatTestMethodsRunInParallelThreads() {
verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5);

verifyParallelTestMethodsWithNonParallelDataProvider(
getTestMethodLevelEventLogsForTest(SUITE, TEST), TEST, expectedInvocationCounts,
5, 5
);

//verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5);
}

//Verifies that all the test method level events for any given test method run in the same thread.
Expand Down
Loading