From 929a37714839a6302e292d9e2ed35bf71cdf026a Mon Sep 17 00:00:00 2001 From: Selena Renee Phillips Date: Fri, 21 Jul 2017 21:16:15 -0400 Subject: [PATCH] Added source code comments to help clarify how the verification of parallelizations is done for methods expected to be executing simultaneously in different threads. Adding logging to help diagnose interimittent failures on TeamCity. Make the execution state tracker final and changed the reset method to remove logs rather than reassign the variable with a new list. Fixed a bug in the calculation of the expected size of the last block of methods to execute in parallel when processing event logs for test methods which do not use data providers without factories. --- .../BaseParallelizationTest.java | 379 ++++++++++++++---- .../ParallelByMethodsTestCase1Scenario1.java | 19 +- .../ParallelByMethodsTestCase1Scenario2.java | 49 +++ .../ParallelByMethodsTestCase2Scenario1.java | 30 +- .../ParallelByMethodsTestCase2Scenario2.java | 49 +++ .../ParallelByMethodsTestCase3Scenario1.java | 51 ++- .../ParallelByMethodsTestCase3Scenario2.java | 31 +- .../ParallelByMethodsTestCase4Scenario1.java | 53 +++ .../ParallelByMethodsTestCase5Scenario1.java | 16 + .../ParallelByMethodsTestCase5Scenario2.java | 35 ++ .../ParallelByMethodsTestCase6Scenario1.java | 51 +++ .../ParallelByMethodsTestCase7Scenario1.java | 22 +- .../ParallelByMethodsTestCase7Scenario2.java | 33 ++ .../ParallelByMethodsTestCase8Scenario1.java | 51 +++ .../TestNgRunStateTracker.java | 39 +- 15 files changed, 826 insertions(+), 82 deletions(-) diff --git a/src/test/java/test/thread/parallelization/BaseParallelizationTest.java b/src/test/java/test/thread/parallelization/BaseParallelizationTest.java index 312e37b8d5..a60591af83 100644 --- a/src/test/java/test/thread/parallelization/BaseParallelizationTest.java +++ b/src/test/java/test/thread/parallelization/BaseParallelizationTest.java @@ -16,6 +16,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import test.thread.parallelization.TestNgRunStateTracker.EventLog; import test.thread.parallelization.TestNgRunStateTracker.TestNgRunEvent; @@ -39,6 +41,15 @@ public class BaseParallelizationTest extends SimpleBaseTest { + private static final Logger logger = Logger.getLogger(BaseParallelizationTest.class.getCanonicalName()); + + { + System.setProperty("java.util.logging.SimpleFormatter.format","%n [%4$s] %2$s - %5$s"); + logger.setLevel(Level.INFO); + } + +// String x = "%1$tF %1$tT"; + //Get a list of the names of declared methods with the @Test annotation from the specified class public static List getDeclaredTestMethods(Class clazz) { List methodNames = new ArrayList<>(); @@ -205,7 +216,7 @@ public static void verifyEventsOccurBetween(EventLog earlierEventLog, List eventLogs, String failMessage) { for (int i = 0; i + 1 < eventLogs.size(); i++) { @@ -303,38 +314,95 @@ public static void verifySameInstancesOfTestClassAssociatedWithMethods(String su } //Verify that methods associated with the specified event logs execute simultaneously in parallel fashion, in - //accordance with the expected maximum number of simultaneous executions. This verification is for blocks of - //parallel methods that have the same sleep delays for their execution bodies and which do not have any - //BeforeMethod AfterMethod, BeforeGroup or AfterGroup configuration methods. + //accordance with the thread count. This verification is for blocks of parallel methods that have the same sleep + //delays for their execution bodies and which do not have any BeforeMethod AfterMethod, BeforeGroup or AfterGroup + //configuration methods. public static void verifySimultaneousTestMethods(List testMethodEventLogs, String testName, int maxSimultaneousTestMethods) { - int remainder = testMethodEventLogs.size() % (maxSimultaneousTestMethods * 3); - + logger.log(Level.INFO,"Verifying parallel execution of test methods for test named {0} with thread count {1}", + new Object[] {testName, maxSimultaneousTestMethods}); + + logger.log(Level.INFO, "{0} test method event logs for {1} test methods: ", + new Object[]{ testMethodEventLogs.size(), testMethodEventLogs.size()/3} ); + + //There are three test method events expected per test method: a start event, an execution event, and a test + //method pass event. All methods take exactly the same amount of time to execute. Each one of their events + //takes exactly the same time to execute. The reason for this is that it makes it possible to assume that blocks + //of methods should execute in parallel in lockstep, starting and finishing at the same time. + // + //The TestNgRunStateListener logs the start and pass events. The test method execution event is logged by the + //test method itself. See the sample test classes for examples. This test method verifies the parallel + //execution of test methods for parallelization tests involving parallel-by-methods mode. Therefore, the + //expectation is that there are simultaneously executing blocks of methods. The'size' of the block is either + //equal to the thread count or less in the event that the total number of methods is not a multiple of the + //thread count and we are processing the final block of methods to execute. + // + //This smaller, last block size is calculated using the number of events logged, the number of events logged + // per method (3) and the thread count to find the number of events expected for that remainder block. + int remainder = (testMethodEventLogs.size() / 3) % maxSimultaneousTestMethods; + + int numBlocks = testMethodEventLogs.size() / 3 < maxSimultaneousTestMethods ? 1 : + (testMethodEventLogs.size() / 3) / maxSimultaneousTestMethods + (remainder > 0 ? 1 : 0); + + log(testMethodEventLogs.size(), maxSimultaneousTestMethods, remainder); + + int loopNum = 1; + + //Loop over the event logs. The increment is equal the thread count times the number of events logged for each + //test method. for (int i = 1; i < testMethodEventLogs.size(); i = i + maxSimultaneousTestMethods * 3) { + + logger.log(Level.INFO, "Processing block {0} of {1}", new Object[] {loopNum, numBlocks}); + + //The size of the block is equal to the thread count or the number of methods left over in the last block + //if the total number of methods is not a multiple of the thread count. Example: For a test run with 19 + //methods in total and a thread count of 7, the remainder is 5 methods. The last block of methods to execute + //would have 5 methods executing simultaneously. If the remainder is non-zero, and we are processing the + //last block of methods then the block size is less than the thread count. Otherwise, the block size is + //equal to the thread count. int blockSize = (remainder != 0 && testMethodEventLogs.size() - i < maxSimultaneousTestMethods * 3) ? - remainder / 3 : + remainder : maxSimultaneousTestMethods; + //The expectation for the block of methods executing in parallel is that the test method start events are + //logged first, then the test method execution events, followed by the test method pass events. These + //offset values are used to extract the sublists of start events, execution events and pass events for the + //block of event logs to process in the current loop execution. int offsetOne = (remainder != 0 && testMethodEventLogs.size() - i < maxSimultaneousTestMethods * 3) ? - testMethodEventLogs.size() - remainder : + testMethodEventLogs.size() - (remainder * 3) : i - 1; int offsetTwo = (remainder != 0 && testMethodEventLogs.size() - i < maxSimultaneousTestMethods * 3) ? - testMethodEventLogs.size() - remainder + blockSize : + testMethodEventLogs.size() - (remainder * 3) + blockSize : i + maxSimultaneousTestMethods - 1; + logger.log(Level.INFO, "Expecting {0} test method start events, followed by {0} test method execution " + + "events, followed by {0} test method pass events", blockSize); + List eventLogMethodListenerStartSublist = testMethodEventLogs.subList(offsetOne, offsetTwo); List eventLogMethodExecuteSublist = testMethodEventLogs.subList(offsetTwo, offsetTwo + blockSize); List eventLogMethodListenerPassSublist = testMethodEventLogs.subList(offsetTwo + blockSize, offsetTwo + 2 * blockSize); + log(offsetOne, offsetTwo, blockSize, eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, + eventLogMethodListenerPassSublist); + + //Verify that all the events in the sublist extracted for the start events of the block of methods expected + //to execute in parallel all have the test method start event type and that they all executed in different + //threads. verifySimultaneousTestMethodListenerStartEvents(eventLogMethodListenerStartSublist, testName, - maxSimultaneousTestMethods); + blockSize); + //Verify that all the events in the sublist extracted for the test method execution events of the block of + //methods expected to execute in parallel all have the test method execution event type and that they all + //executed in different threads. verifySimultaneousTestMethodExecutionEvents(eventLogMethodExecuteSublist, testName, - maxSimultaneousTestMethods); + blockSize); + //Verify that the test method start events and the test method execution events in the two sublists belong + //to the same methods. This is done by verifying that the test class names and method names are the same for + //for both sublists. verifyEventsBelongToSameMethods(eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, "The " + "expected maximum number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + maxSimultaneousTestMethods + " methods should be " + @@ -342,42 +410,121 @@ public static void verifySimultaneousTestMethods(List testMethodEventL "test methods should all belong to the same methods as the test method listener onTestStart " + "event logs immediately preceding"); + //Verify that all the events in the sublist extracted for the test method pass events of the block of + //methods expected to execute in parallel all have the test method pass event type and that they all + //executed in different threads. verifySimultaneousTestMethodListenerPassEvents(eventLogMethodListenerPassSublist, testName, - maxSimultaneousTestMethods); + blockSize); + //Verify that the test method execution events and the test method pass events in the two sublists belong + //to the same methods. This is done by verifying that the test class names and method names are the same for + //for both sublists. verifyEventsBelongToSameMethods(eventLogMethodExecuteSublist, eventLogMethodListenerPassSublist, "The " + "expected maximum number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + maxSimultaneousTestMethods + " methods should be " + "running at the same time. The test method listener on onTestSuccess event logs for a block of " + "simultaneously running test methods should all belong to the same methods as the test method " + "execution event logs immediately preceding"); + + loopNum++; } } + //Verify that methods associated with the specified event logs execute simultaneously in parallel fashion, in + //accordance with the thread count. This verification is for blocks of parallel methods, some of which use + //non-parallel data providers without factories, so all the invocations of the methods run on the same class + //instances. This verification is for blocks of parallel methods that have the same sleep delays for their + //execution bodies and which do not have any BeforeMethod AfterMethod, BeforeGroup or AfterGroup + //configuration methods. public static void verifyParallelTestMethodsWithNonParallelDataProvider(List testMethodEventLogs, String testName, Map expectedInvocationCounts, int numUniqueMethods, int maxSimultaneousTestMethods) { + logger.log(Level.INFO,"Verifying parallel execution of test methods using non-parallel data providers for " + + "test named {0} with thread count {1}", new Object[] {testName, maxSimultaneousTestMethods}); + + logger.log(Level.INFO, "{0} test method event logs for {1} unique methods: ", + new Object[] {testMethodEventLogs.size(), numUniqueMethods}); + + //Some of the test methods use non-parallel data providers without factories. All the invocations of those + //test methods will occur serially within the same thread on the same class instances. In order to ensure that + //the loop logic below works properly, it is necessary to keep state information about which methods are + //supposed to be executing within a block of parallel methods that are running simultaneously. Unlike the + //logic in verifySimultaneousTestMethods, it is not possible to assume that methods within a block of + //simultaneously executing methods start and finish at the same time because the methods will frequently be + //invoked a varying number of times, depending on their use of data providers. + // + //However, each _invocation_ of a test method should take exactly the same amount of time. There are three test + //method events expected per test method: a start event, an execution event, and a test method pass event. All + //test method events of the same time take the same amount of time to execute. The reason for this is that it + //makes it possible to assume that blocks of method invocations should execute in parallel in lockstep, + //starting and finishing at the same time. Map methodsExecuting = new HashMap<>(); + + //This isn't actually used for any verification logic. I may remove it in the future. Map methodsCompleted = new HashMap<>(); + + //Because this method verifies combination of parallel-by-methods mode and the use of non-parallel data + //providers without factories, it is necessary to track the number of times that test methods are invoked in + //order to check that this is consistent with the number of times they are expected to be invoked based on + //their use of non-parallel data providers. Map methodInvocationsCounts = new HashMap<>(); + + //In order to verify that all invocations of test methods which use non-parallel data providers occur in the + //same thread, it is necessary to keep track of the thread IDs of methods that are executing within a + //block of simultaneously executing methods. Map executingMethodThreadIds = new HashMap<>(); + //The logic for determining the block size of simultaneously executing parallel methods is initially determined + //by whether the total number of unique methods less than the thread count. If it is less than the thread count, + //then the block size is equal to the number of unique methods. Those methods will execute in parallel + //until all invocations of all the methods completes. Otherwise, there are more methods queued up than the + //thread count, so the block size is equal to the thread count. int blockSize = numUniqueMethods >= maxSimultaneousTestMethods ? maxSimultaneousTestMethods : numUniqueMethods; + int loopNum = 1; + for (int i = 1; i < testMethodEventLogs.size(); i = i + blockSize * 3) { + logger.log(Level.INFO, "Processing block {0}", loopNum); + + //If the loop is executing more than once, then the block size needs to be updated. The number of remaining + //unique methods to execute determines the block size of parallel methods expected to execute + //simultaneously. if(i != 1) { + + //All methods that are in the list of currently executing methods should still have invocations left. + //Otherwise, they would have been removed from that list and added to the completed methods list. + allExecutingMethodsHaveMoreInvocations(methodsExecuting, methodInvocationsCounts, + expectedInvocationCounts); + + //If there are no remaining unique methods, the block size of methods expected to be executing in + //parallel is equal to the number of methods using data providers that are already executing. The only + //test method event logs to verify in this loop should belong to test methods which use data providers + //and are executed multiple times as a result. if(numUniqueMethods == 0) { blockSize = methodsExecuting.keySet().size(); - } else if(numUniqueMethods < maxSimultaneousTestMethods) { - if(methodsExecuting.keySet().size() == maxSimultaneousTestMethods && - allExecutingMethodsHaveMoreInvocations(methodsExecuting, methodInvocationsCounts, - expectedInvocationCounts)) { - blockSize = methodsExecuting.keySet().size(); + } else { + //Otherwise, if the number of unique methods left is non-zero, but less than the thread count, + //the block size is dependent on whether the number of currently executing methods is equal to the + //thread count. If so, then the block size is equal to the thread count and no new unique methods + //will begin executing this block of methods. If the number of methods already executing is less + //than the thread count and the sum of the number of unique methods left and the number of currently + //executing methods is equal to or greater than the thread count, the block size is equal to the + //thread count. If the sum is less than the thread count, the block size is equal to the sum. + if(numUniqueMethods < maxSimultaneousTestMethods) { + + if (methodsExecuting.keySet().size() == maxSimultaneousTestMethods || + numUniqueMethods + methodsExecuting.keySet().size() >= maxSimultaneousTestMethods) { + blockSize = maxSimultaneousTestMethods; + } else { + blockSize = numUniqueMethods + methodsExecuting.keySet().size(); + } + //If the number of unique methods left is more than or equal to the thread count, the block size + //is equal to the thread count. } else { - blockSize = numUniqueMethods + methodsExecuting.keySet().size(); + blockSize = maxSimultaneousTestMethods; } } } @@ -386,21 +533,39 @@ public static void verifyParallelTestMethodsWithNonParallelDataProvider(List eventLogMethodListenerStartSublist = testMethodEventLogs.subList(offsetOne, offsetTwo); List eventLogMethodExecuteSublist = testMethodEventLogs.subList(offsetTwo, offsetTwo + blockSize); List eventLogMethodListenerPassSublist = testMethodEventLogs.subList(offsetTwo + blockSize, offsetTwo + 2 * blockSize); + log(offsetOne, offsetTwo, blockSize, eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, + eventLogMethodListenerPassSublist); + + //Verify that all the events in the sublist extracted for the start events of the block of methods expected + //to execute in parallel all have the test method start event type and that they all executed in different + //threads. The method should return the total number of new unique methods that began executing in the + //current block of parallel methods executing in parallel. int decrementUniqueMethods = verifySimultaneousTestMethodListenerStartEvents( - eventLogMethodListenerStartSublist, testName, maxSimultaneousTestMethods, methodsExecuting, + eventLogMethodListenerStartSublist, testName, blockSize, methodsExecuting, executingMethodThreadIds, methodInvocationsCounts, expectedInvocationCounts ); + //Decrement the unique of unique methods left to execute by the number of new unique methods that began + //execution in this block of parallel methods. numUniqueMethods = numUniqueMethods - decrementUniqueMethods; + //Verify that all the events in the sublist extracted for the test method execution events of the block of + //methods expected to execute in parallel all have the test method execution event type and that they all + //executed in different threads. verifySimultaneousTestMethodExecutionEvents(eventLogMethodExecuteSublist, testName, - executingMethodThreadIds, maxSimultaneousTestMethods); + executingMethodThreadIds, blockSize); + //Verify that the test method start events and the test method execution events in the two sublists belong + //to the same methods. This is done by verifying that the test class names and method names are the same for + //for both sublists. verifyEventsBelongToSameMethods(eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, "The " + "expected maximum number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + maxSimultaneousTestMethods + " methods should be " + @@ -408,10 +573,22 @@ public static void verifyParallelTestMethodsWithNonParallelDataProvider(List listenerStartEventLogs, String - testName, int maxSimultaneousTestMethods, Map methodsExecuting, Map + testName, int blockSize, Map methodsExecuting, Map executingMethodThreadIds, Map methodInvocationsCounts, Map expectedInvocationCounts) { - verifySimultaneousTestMethodListenerStartEvents(listenerStartEventLogs, testName, maxSimultaneousTestMethods); + verifySimultaneousTestMethodListenerStartEvents(listenerStartEventLogs, testName, blockSize); int decrement = 0; @@ -453,10 +631,12 @@ public static int verifySimultaneousTestMethodListenerStartEvents(List if (methodsExecuting.keySet().contains(classAndMethodName)) { assertTrue(eventLog.getThreadId() == executingMethodThreadIds.get(classAndMethodName), "All " + "invocations of method '" + classAndMethodName + "' should execute in the same " + - "thread"); + "thread, but some event logs have different thread IDs"); } else { assertFalse(executingMethodThreadIds.values().contains(eventLog.getThreadId()), "Event logs " + - "for currently executing methods should have different thread IDs: " + classAndMethodName); + "for different methods currently executing should have different thread IDs, but some event " + + "logs for different methods in the current block being processed have the same thread ID: " + + classAndMethodName); } if(methodsExecuting.get(classAndMethodName) == null) { @@ -469,33 +649,37 @@ public static int verifySimultaneousTestMethodListenerStartEvents(List } public static void verifySimultaneousTestMethodExecutionEvents(List testMethodExecutionEventLogs, String - testName, Map executingMethodThreadIds, int maxSimultaneousTestMethods) { + testName, Map executingMethodThreadIds, int blockSize) { - verifySimultaneousTestMethodExecutionEvents(testMethodExecutionEventLogs, testName, maxSimultaneousTestMethods); + verifySimultaneousTestMethodExecutionEvents(testMethodExecutionEventLogs, testName, blockSize); for(EventLog eventLog : testMethodExecutionEventLogs) { String classAndMethodName = (String)eventLog.getData(CLASS_NAME) + "." + (String)eventLog.getData(METHOD_NAME); assertTrue(eventLog.getThreadId() == executingMethodThreadIds.get(classAndMethodName), "All the " + - "test method event logs for a given method should have the same thread ID"); + "test method event logs for a given method using a non-parallel data provider should have the " + + "same thread ID, but some event logs for a method have different thread IDs: " + + classAndMethodName); } } public static void verifySimultaneousTestMethodListenerPassEvents(List testMethodListenerPassEventLogs, - String testName, int maxSimultaneousTestMethods, Map methodsExecuting, + String testName, int blockSize, Map methodsExecuting, Map methodsCompleted, Map executingMethodThreadIds, Map methodInvocationsCounts, Map expectedInvocationCounts) { verifySimultaneousTestMethodListenerPassEvents(testMethodListenerPassEventLogs, testName, - maxSimultaneousTestMethods); + blockSize); for(EventLog eventLog : testMethodListenerPassEventLogs) { String classAndMethodName = (String)eventLog.getData(CLASS_NAME) + "." + (String)eventLog.getData(METHOD_NAME); assertTrue(eventLog.getThreadId() == executingMethodThreadIds.get(classAndMethodName), "All the " + - "test method event logs for a given method should have the same thread ID"); + "test method event logs for a given method using a non-parallel data provider should have the " + + "same thread ID, but some event logs for a method have different thread IDs: " + + classAndMethodName); if(methodInvocationsCounts.get(classAndMethodName) .equals(expectedInvocationCounts.get(classAndMethodName))) { @@ -511,48 +695,45 @@ public static void verifySimultaneousTestMethodListenerPassEvents(List //according to the expected maximum number of simultaneous executions. Verifies that each of them has the same //event type and all have different thread IDs. public static void verifySimultaneousTestMethodListenerStartEvents(List listenerStartEventLogs, String - testName, int maxSimultaneousTestMethods) { + testName, int blockSize) { + + verifyEventTypeForEventsLogs(listenerStartEventLogs, LISTENER_TEST_METHOD_START, "Expected " + blockSize + + " test method start event logs to be in a block of methods executing in parallel. Found an event log " + + "of a different type in the block being processed: " + listenerStartEventLogs); - verifyEventTypeForEventsLogs(listenerStartEventLogs, LISTENER_TEST_METHOD_START, "The expected maximum " + - "number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + - " so more no more than " + maxSimultaneousTestMethods + " methods should start running at the same " + - "time if there are more than " + maxSimultaneousTestMethods + " methods remaining to execute. Event " + - "logs: " + listenerStartEventLogs); - verifyDifferentThreadIdsForEvents(listenerStartEventLogs, "The expected maximum number of methods to execute " + - "simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so the thread IDs for all " + - "the test method listener's onTestStart method " + "the " + maxSimultaneousTestMethods + "currently " + - "executing test methods should be different. Event logs: " + listenerStartEventLogs); + verifyDifferentThreadIdsForEvents(listenerStartEventLogs, "Expected " + blockSize + " test method start " + + "event logs to be in a block of methods executing in parallel. Each one of these event logs should " + + "be associated with a different thread ID, but found that at two event logs share the same thread " + + "ID: " + listenerStartEventLogs); } //Verify that the specified test method execution event logs execute simultaneously in parallel fashion according //to the specified thread count. Verifies that each of them has the same event type and all have different thread //IDs. public static void verifySimultaneousTestMethodExecutionEvents(List testMethodExecutionEventLogs, - String testName, int maxSimultaneousTestMethods) { - - verifyEventTypeForEventsLogs(testMethodExecutionEventLogs, TEST_METHOD_EXECUTION, "The expected maximum " + - "number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + - " so no more than " + maxSimultaneousTestMethods + " methods should be " + "executing at the same " + - "time. Event logs: " + testMethodExecutionEventLogs); - verifyDifferentThreadIdsForEvents(testMethodExecutionEventLogs, "The expected maximum number of methods to " + - "execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so the thread IDs " + - "for the test method execution events for the " + maxSimultaneousTestMethods + "currently executing " + - "test methods should be different. Event logs: " + testMethodExecutionEventLogs); + String testName, int blockSize) { + + verifyEventTypeForEventsLogs(testMethodExecutionEventLogs, TEST_METHOD_EXECUTION, "Expected " + blockSize + + " test method execution event logs to be in a block of methods executing in parallel. Found an event " + + "log of a different type in the block being processed: " + testMethodExecutionEventLogs); + verifyDifferentThreadIdsForEvents(testMethodExecutionEventLogs, "Expected " + blockSize + " test method " + + "execution event logs to be in a block of methods executing in parallel. Each one of these event " + + "logs should be associated with a different thread ID, but found that at two event logs share the " + + "same thread ID: " + testMethodExecutionEventLogs); } //Verify that the specified test method listener onTestSuccess event logs execute simultaneously in parallel //fashion according to the specified thread count. Verifies that each of them has the same event type and all have //different thread IDs. public static void verifySimultaneousTestMethodListenerPassEvents(List testMethodListenerPassEventLogs, - String testName, int maxSimultaneousTestMethods) { - verifyEventTypeForEventsLogs(testMethodListenerPassEventLogs, LISTENER_TEST_METHOD_PASS, "The thread " + - "count is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + - maxSimultaneousTestMethods + " test listener " + "onTestSuccess methods should be executing at the " + - "same time. Event logs: " + testMethodListenerPassEventLogs); - verifyDifferentThreadIdsForEvents(testMethodListenerPassEventLogs, "The expected maximum number of methods " + - "to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so the thread " + - "IDs for the test method listener onTestSuccess events for the " + maxSimultaneousTestMethods + - "currently executing test methods should be different. Event logs: " + testMethodListenerPassEventLogs); + String testName, int blockSize) { + verifyEventTypeForEventsLogs(testMethodListenerPassEventLogs, LISTENER_TEST_METHOD_PASS, "Expected " + + blockSize + " test method pass event logs to be in a block of methods executing in parallel. Found " + + "an event log of a different type in the block being processed: " + testMethodListenerPassEventLogs); + verifyDifferentThreadIdsForEvents(testMethodListenerPassEventLogs, "Expected " + blockSize + " test method " + + "pass event logs to be in a block of methods executing in parallel. Each one of these event " + + "logs should be associated with a different thread ID, but found that at two event logs share the " + + "same thread ID: " + testMethodListenerPassEventLogs); } //Verify that the test method level events for the test methods declared in the specified class run in the same @@ -627,7 +808,7 @@ public static void verifySequentialSuites(List suiteLevelEventLogs, Ma assertTrue(suiteLevelEventLogs.get(i).getEvent() == LISTENER_SUITE_START && suiteLevelEventLogs.get(i + 1).getEvent() == LISTENER_SUITE_FINISH, "Because the suites are " + "expected to execute sequentially, the suite level event logs should consist of a series of " + - "pairs of a suite listener onStart event log followed by a suite listener onFinish event log: " + + "pairs of a suite listener onStart event logger followed by a suite listener onFinish event logger: " + suiteLevelEventLogs); suiteListenerStartEventLogs.add((suiteLevelEventLogs.get(i))); } @@ -640,7 +821,7 @@ public static void verifySequentialSuites(List suiteLevelEventLogs, Ma List secondSuiteEventLogs = suiteEventLogsMap.get(secondSuite); verifySequentialTimingOfEvents(firstSuiteEventLogs, secondSuiteEventLogs, "The first suite listener " + - "onStart event log is for " + firstSuite + " and the second suite listener onStart event log is " + + "onStart event logger is for " + firstSuite + " and the second suite listener onStart event logger is " + "for " + secondSuite + ". Because the suites are supposed to execute sequentially, all of the " + "event logs for " + firstSuite + " should have timestamps earlier than all of the event logs for " + secondSuite + ". First suite event logs: " + firstSuiteEventLogs + ". Second suite event logs: " + @@ -661,15 +842,15 @@ public static void verifySequentialTests(List suiteAndTestLevelEventLo verifyEventsOccurBetween(suiteListenerOnStartEventLog, testLevelEventLogs, suiteListenerOnFinishEventLog, "All of the test level event logs should have timestamps between the suite listener's onStart and " + - "onFinish event logs. Suite listener onStart event log: " + suiteListenerOnStartEventLog + - ". Suite listener onFinish event log: " + suiteListenerOnFinishEventLog + ". Test level " + + "onFinish event logs. Suite listener onStart event logger: " + suiteListenerOnStartEventLog + + ". Suite listener onFinish event logger: " + suiteListenerOnFinishEventLog + ". Test level " + "event logs: " + testLevelEventLogs); for (int i = 0; i < testLevelEventLogs.size(); i = i + 2) { assertTrue(testLevelEventLogs.get(i).getEvent() == LISTENER_TEST_START && testLevelEventLogs.get(i + 1).getEvent() == LISTENER_TEST_FINISH, "Because the tests are " + "expected to execute sequentially, the test level event logs should consist of a series of " + - "pairs of a test listener onStart event log followed by a test listener onFinish event log: " + + "pairs of a test listener onStart event logger followed by a test listener onFinish event logger: " + testLevelEventLogs); } } @@ -733,8 +914,8 @@ public static void verifyParallelSuitesWithUnequalExecutionTimes(List if (eventLog.getEvent() == LISTENER_SUITE_FINISH) { - assertTrue(suitesExecuting.get(suiteName) != null, "Found an event log for a suite listener " + - "onFinish event that does not have a corresponding event log for a suite listener " + + assertTrue(suitesExecuting.get(suiteName) != null, "Found an event logger for a suite listener " + + "onFinish event that does not have a corresponding event logger for a suite listener " + "onStart event"); assertTrue(suitesExecuting.get(suiteName).getThreadId() == eventLog.getThreadId(), "All the " + "suite level event logs for a given suite should have the same thread ID"); @@ -821,5 +1002,69 @@ private static boolean allExecutingMethodsHaveMoreInvocations(Map eventLogMethodListenerStartSublist, + List eventLogMethodExecuteSublist, List eventLogMethodListenerPassSublist) { + logger.log(Level.INFO, "Event logs extracted from event log list between index {0} and index {1} should " + + "be the test method start event logs for a block of {2} simultaneously executing methods", + new Object[] {offsetOne, offsetTwo - 1, blockSize}); + + int j = offsetOne; + + for(EventLog eventLog : eventLogMethodListenerStartSublist) { + logger.log(Level.INFO, "Event logged at index {0}: {1}", new Object[] {j, eventLog.toString()}); + j++; + } + + logger.log(Level.INFO, "Event logs extracted from event log list between index {0} and index {1} should " + + "be the test method execution event logs for a block of {2} simultaneously executing methods", + new Object[] {offsetTwo, offsetTwo + blockSize - 1, blockSize}); + + j = offsetTwo; + + for(EventLog eventLog : eventLogMethodExecuteSublist) { + logger.log(Level.INFO, "Event logged at index {0}: {1}", new Object[] {j, eventLog.toString()}); + j++; + } + + logger.log(Level.INFO, "Event logs extracted from event log list between index {0} and index {1} should " + + "be the test method pass event logs for a block of {2} simultaneously executing methods", + new Object[] {offsetTwo + blockSize, offsetTwo + 2 * blockSize - 1, blockSize}); + + j = offsetTwo + blockSize; + + for(EventLog eventLog : eventLogMethodListenerPassSublist) { + logger.log(Level.INFO, "Event logged at index {0}: {1}", new Object[] {j, eventLog.toString()}); + j++; + } + } + + private static void log(int listSize, int threadCount, int remainder) { + if(listSize / 3 < threadCount) { + logger.log(Level.INFO, "Expecting there to be a single block of {0} parallel methods", listSize / 3); + } else { + + if(remainder > 0) { + logger.log(Level.INFO, "Expecting there to be a series of {0} blocks of {1} parallel methods with a " + + "final block of {2} parallel methods", + + new Object[] + { + (listSize / 3) / threadCount, + threadCount, + remainder + } + ); + } else { + logger.log(Level.INFO, "Expecting there to be a series of {0} blocks of {1} parallel methods", + new Object[] + { + (listSize / 3) / threadCount, + threadCount + } + ); + } + } + } } diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java index 15f6809d4c..0549c6fe4c 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java @@ -1,17 +1,19 @@ package test.thread.parallelization; -import com.google.common.collect.Multimap; import org.testng.ITestNGListener; import org.testng.TestNG; + import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; + import org.testng.xml.XmlSuite; import test.thread.parallelization.TestNgRunStateTracker.EventLog; import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; -import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -46,6 +48,12 @@ */ public class ParallelByMethodsTestCase1Scenario1 extends BaseParallelizationTest { + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -77,6 +85,13 @@ public void setUp() { TestNG tng = create(suite); tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario1. This test scenario consists of a " + + "single suite with a single test which consists of one test class with five test methods. There " + + "are no dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName(), 5}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java index 8ec9a35cba..84183f780b 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java @@ -11,6 +11,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import test.thread.parallelization.TestNgRunStateTracker.EventLog; @@ -77,6 +79,13 @@ * 10) There are no method exclusions */ public class ParallelByMethodsTestCase1Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -208,6 +217,46 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo, suiteThree); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario2. This test scenario consists of three " + + "sequentially executed suites with 1, 2 and 3 tests respectively. One test for a suite consists of a " + + "single test class while the rest shall consist of more than one test class. There are no " + + "dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java index b8e81179b9..191dc9447a 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java @@ -2,8 +2,10 @@ import org.testng.ITestNGListener; import org.testng.TestNG; + import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; + import org.testng.xml.XmlSuite; import test.thread.parallelization.TestNgRunStateTracker.EventLog; @@ -19,6 +21,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -62,6 +66,12 @@ */ public class ParallelByMethodsTestCase2Scenario1 extends BaseParallelizationTest { + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -125,7 +135,6 @@ public void setUp() { suiteTwo.setParallel(XmlSuite.ParallelMode.METHODS); suiteTwo.setThreadCount(14); - addParams(suiteOne, SUITE_A, SUITE_A_TEST_A, "100"); addParams(suiteTwo, SUITE_B, SUITE_B_TEST_A, "100"); @@ -135,6 +144,25 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario1. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. The suites run in parallel and the thread pool size is 2. " + + "One test for a suite shall consist of a single test class while the rest shall consist of more than " + + "one test class. There are no dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 14}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 14}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java index 17374fb298..4165205403 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java @@ -26,6 +26,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -71,6 +73,13 @@ * 11) There are no method exclusions */ public class ParallelByMethodsTestCase2Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -204,6 +213,46 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario2. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is 3 " + + "One test for a suite shall consist of a single test class while the rest shall consist of more than " + + "one test class. There are no dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java index ab15cd6a99..ff94dc8fab 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java @@ -10,8 +10,13 @@ import test.thread.parallelization.TestNgRunStateTracker.EventLog; import test.thread.parallelization.sample.TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample; +import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -25,6 +30,7 @@ import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartEventLog; import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartThreadId; +import static test.thread.parallelization.TestNgRunStateTracker.getTestMethodLevelEventLogsForTest; import static test.thread.parallelization.TestNgRunStateTracker.reset; /** @@ -48,6 +54,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase3Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -64,6 +77,8 @@ public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest private Long testListenerOnStartThreadId; + private Map expectedInvocationCounts = new HashMap<>(); + @BeforeClass public void setUp() { reset(); @@ -72,6 +87,26 @@ public void setUp() { suite.setParallel(XmlSuite.ParallelMode.METHODS); suite.setThreadCount(15); + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodA", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodB", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodC", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodD", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodE", 3); + createXmlTest(suite, TEST, TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class); addParams(suite, SUITE, TEST, "100", "paramOne,paramTwo,paramThree"); @@ -80,6 +115,14 @@ public void setUp() { tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase3Scenario1. This test scenario consists of a " + + "single suite with a single test consisting of a single test class with five methods with a data " + + "provider specifying 3 sets of data. There are no dependencies or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST, + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName(), 15}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); @@ -168,7 +211,13 @@ public void verifyThatMethodLevelEventsRunInDifferentThreadsFromSuiteAndTestLeve //Verifies that the test methods execute in different threads in parallel fashion. @Test public void verifyThatTestMethodsRunInParallelThreads() { - verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5); + + verifyParallelTestMethodsWithNonParallelDataProvider( + getTestMethodLevelEventLogsForTest(SUITE, TEST), TEST, expectedInvocationCounts, + 5, 5 + ); + + //verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5); } //Verifies that all the test method level events for any given test method run in the same thread. diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java index d71fc62e8a..41f18f67b2 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java @@ -17,6 +17,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -60,6 +62,12 @@ */ public class ParallelByMethodsTestCase3Scenario2 extends BaseParallelizationTest { + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase3Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -123,8 +131,8 @@ public void setUp() { TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class); suiteOne.setParallel(XmlSuite.ParallelMode.METHODS); - suiteOne.setThreadCount(3); + suiteTwo.setParallel(XmlSuite.ParallelMode.METHODS); suiteTwo.setThreadCount(4); @@ -137,6 +145,27 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase3Scenario2. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. One test for a suite shall consist of a single test class " + + "while the rest shall consist of more than one test class. Each test class has some methods with use " + + "a data provider and some which do not. Two data providers are used: one which provides two sets of " + + "data, one which provide three sets of data. There are no dependencies or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassBSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A, + TestClassCFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 4}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassEFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 4}); + tng.run(); expectedInvocationCounts.put( diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java index 0d346266e2..028e486cbe 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java @@ -27,6 +27,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllEventLogsForSuite; @@ -78,6 +80,13 @@ * 11) There are no method exclusions */ public class ParallelByMethodsTestCase4Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase4Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -224,6 +233,50 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase4Scenario1. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is " + + "2. One test for a suite shall consist of a single test class while the rest shall consist of more " + + "han one test class. Some test classes have a mixture of some methods that use a data provider and " + + "some which do not. The data providers provide data sets of varying sizes. There are no " + + "dependencies or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassBSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A, + TestClassCFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 6}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class + ", " + + TestClassEFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class + ", " + + TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassDThreeMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassGFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassBFourMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassIThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassJFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassBSixMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName(), + 12}); + tng.run(); expectedInvocationCounts.put( diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java index 7c5fc07422..2c00a433cc 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java @@ -14,6 +14,8 @@ import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteAndTestLevelEventLogs; @@ -48,6 +50,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase5Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase5Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -80,6 +89,13 @@ public void setUp() { tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase5Scenario1. This test scenario consists of a " + + "single suite with a single test consisting of a factory that provides two instances of a single\n" + + "test class with five methods. There are no dependencies or data providers."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST, TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName(), 15}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java index 17539f7ab1..c3433a4566 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java @@ -13,17 +13,25 @@ import test.thread.parallelization.sample.FactoryForTestClassDThreeMethodsWithNoDepsFourInstancesSample; import test.thread.parallelization.sample.FactoryForTestClassFSixMethodsWithNoDepsSixInstancesSample; +import test.thread.parallelization.sample.TestClassAFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassBFourMethodsWithNoDepsSample; +import test.thread.parallelization.sample.TestClassBSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; +import test.thread.parallelization.sample.TestClassCFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassCSixMethodsWithNoDepsSample; +import test.thread.parallelization.sample.TestClassDThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassDThreeMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassEFiveMethodsWithNoDepsSample; +import test.thread.parallelization.sample.TestClassEFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; +import test.thread.parallelization.sample.TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassFSixMethodsWithNoDepsSample; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllEventLogsForSuite; @@ -66,6 +74,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase5Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase5Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -145,6 +160,26 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase5Scenario2. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. One suite with two tests has a test consisting of a single " + + "test class without a factory while the other consists of factories which provide multiple instances " + + "of multiple test classes. One suite shall consist of a single test with multiple test classes which " + + "uses factories. There are no dependencies or data providers."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java index fd97149c8a..32be90d6cb 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java @@ -31,6 +31,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteLevelEventLogs; @@ -80,6 +82,13 @@ * 12) There are no method exclusions */ public class ParallelByMethodsTestCase6Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase6Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -214,6 +223,48 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase6Scenario1. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is " + + "2. One suite with two tests has a test consisting of a single test class without a factory while " + + "the other shall consist of factories which provide multiple instances of multiple test classes. One " + + "suite shall consist of a single test with multiple test classes which uses factories. One suite " + + "shall have multiple tests with multiple classes, none of which use a factory. There are no " + + "dependencies."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java index 37e159b9ce..1db213cac4 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java @@ -12,8 +12,11 @@ import test.thread.parallelization.sample.TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; + import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteAndTestLevelEventLogs; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteLevelEventLogs; import static test.thread.parallelization.TestNgRunStateTracker.getAllTestLevelEventLogs; @@ -24,6 +27,7 @@ import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartEventLog; import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartThreadId; +import static test.thread.parallelization.TestNgRunStateTracker.getTestMethodLevelEventLogsForTest; import static test.thread.parallelization.TestNgRunStateTracker.reset; /** This class covers PTP_TC_7, Scenario 1 in the Parallelization Test Plan. @@ -48,6 +52,13 @@ * 8) There are no method exclusions */ public class ParallelByMethodsTestCase7Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase7Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -80,6 +91,14 @@ public void setUp() { tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase7Scenario1. This test scenario consists of a " + + "single suite with a single test consisting of a single test class with five methods with a " + + "factory method using a data provider specifying 3 sets of data. There are no dependencies."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST, + TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName(), 15}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); @@ -176,7 +195,6 @@ public void verifyThatAllEventsForATestMethodExecuteInSameThread() { //Verifies that the test methods execute in different threads in parallel fashion. @Test public void verifyThatTestMethodsRunInParallelThreads() { - verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 15); + verifySimultaneousTestMethods(getTestMethodLevelEventLogsForTest(SUITE, TEST), TEST, 15); } - } diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java index 03e6981dcb..ddbe982554 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java @@ -8,9 +8,11 @@ import org.testng.xml.XmlTest; import test.thread.parallelization.sample.TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample; +import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassBFourMethodsWithFactoryUsingDataProviderAndNoDepsSample; import test.thread.parallelization.sample.TestClassBFourMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassCSixMethodsWithFactoryUsingDataProviderAndNoDepsSample; +import test.thread.parallelization.sample.TestClassCSixMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassDThreeMethodsWithFactoryUsingDataProviderAndNoDepsSample; import test.thread.parallelization.sample.TestClassDThreeMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassEFiveMethodsWithNoDepsSample; @@ -21,6 +23,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -65,6 +69,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase7Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase7Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -157,6 +168,28 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase7Scenario2. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. One suite with two tests has a test consisting of a single " + + "test class without a factory while the other consists of factories using data providers with " + + "varying numbers of data sets which provide multiple instances of multiple test classes. One suite " + + "shall consist of a single test with multiple test classes which use factories with data providers " + + "with varying numbers of data sets. There are no dependencies."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName(), 25}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassBFourMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassFSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class, + 40}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java index b963d583b0..a92f458a1f 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java @@ -35,6 +35,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteLevelEventLogs; @@ -86,6 +88,13 @@ * 12) here are no method exclusions */ public class ParallelByMethodsTestCase8Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase8Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -227,6 +236,48 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase8Scenario1. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. One suite with two tests has a test consisting of a " + + "single test class without a factory while the other shall consist of test classes with factories " + + "using data providers with varying numbers of data sets. One suite shall consist of a single test " + + "with multiple test classes with factories using data providers with varying numbers of data sets. " + + "One suite shall have multiple tests with multiple classes, none of which use a factory."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName(), 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassBFourMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassFSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); tng.run(); expectedInvocationCounts.put( diff --git a/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java b/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java index 706424a2db..37c2a3c7af 100644 --- a/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java +++ b/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java @@ -16,7 +16,7 @@ */ public class TestNgRunStateTracker { - private static List eventLogs = new ArrayList<>(); + private static final List eventLogs = new ArrayList<>(); public static void logEvent(EventLog eventLog) { synchronized(eventLogs) { @@ -754,7 +754,7 @@ public static Map getTestMethodExecutionThreadIds(String suiteName public static void reset() { - eventLogs = new ArrayList<>(); + eventLogs.clear(); } private static boolean isSuiteLevelEventLog(EventLog eventLog) { @@ -872,12 +872,35 @@ public static EventLogBuilder builder() { @Override public String toString() { - return "EventLog{" + - "event=" + event + - ", timeOfEvent=" + timeOfEvent + - ", threadId=" + threadId + - ", data=" + data + - '}'; + final StringBuffer sb = new StringBuffer("EventLog{"); + sb.append("Event: ").append(event); + + sb.append(", Suite: ").append(getData(EventInfo.SUITE_NAME)); + + if(getData(EventInfo.TEST_NAME) != null) { + sb.append(", Test: ").append(getData(EventInfo.TEST_NAME)); + } + + if(getData(EventInfo.CLASS_NAME) != null) { + sb.append(", Class: ").append(getData(EventInfo.CLASS_NAME)); + } + + if(getData(EventInfo.CLASS_INSTANCE) != null) { + sb.append(", Class instance hash code: ").append(getData(EventInfo.CLASS_INSTANCE).hashCode()); + } + + if(getData(EventInfo.METHOD_NAME) != null) { + sb.append(", Method name: ").append(getData(EventInfo.METHOD_NAME)); + } + + if(getData(EventInfo.DATA_PROVIDER_PARAM) != null) { + sb.append(", Data provider param: ").append(getData(EventInfo.DATA_PROVIDER_PARAM)); + } + + sb.append(", Time of event: ").append(timeOfEvent); + sb.append(", Thread ID: ").append(threadId); + sb.append("}"); + return sb.toString(); } }