diff --git a/src/test/java/test/thread/parallelization/BaseParallelizationTest.java b/src/test/java/test/thread/parallelization/BaseParallelizationTest.java index 312e37b8d5..a60591af83 100644 --- a/src/test/java/test/thread/parallelization/BaseParallelizationTest.java +++ b/src/test/java/test/thread/parallelization/BaseParallelizationTest.java @@ -16,6 +16,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import test.thread.parallelization.TestNgRunStateTracker.EventLog; import test.thread.parallelization.TestNgRunStateTracker.TestNgRunEvent; @@ -39,6 +41,15 @@ public class BaseParallelizationTest extends SimpleBaseTest { + private static final Logger logger = Logger.getLogger(BaseParallelizationTest.class.getCanonicalName()); + + { + System.setProperty("java.util.logging.SimpleFormatter.format","%n [%4$s] %2$s - %5$s"); + logger.setLevel(Level.INFO); + } + +// String x = "%1$tF %1$tT"; + //Get a list of the names of declared methods with the @Test annotation from the specified class public static List getDeclaredTestMethods(Class clazz) { List methodNames = new ArrayList<>(); @@ -205,7 +216,7 @@ public static void verifyEventsOccurBetween(EventLog earlierEventLog, List eventLogs, String failMessage) { for (int i = 0; i + 1 < eventLogs.size(); i++) { @@ -303,38 +314,95 @@ public static void verifySameInstancesOfTestClassAssociatedWithMethods(String su } //Verify that methods associated with the specified event logs execute simultaneously in parallel fashion, in - //accordance with the expected maximum number of simultaneous executions. This verification is for blocks of - //parallel methods that have the same sleep delays for their execution bodies and which do not have any - //BeforeMethod AfterMethod, BeforeGroup or AfterGroup configuration methods. + //accordance with the thread count. This verification is for blocks of parallel methods that have the same sleep + //delays for their execution bodies and which do not have any BeforeMethod AfterMethod, BeforeGroup or AfterGroup + //configuration methods. public static void verifySimultaneousTestMethods(List testMethodEventLogs, String testName, int maxSimultaneousTestMethods) { - int remainder = testMethodEventLogs.size() % (maxSimultaneousTestMethods * 3); - + logger.log(Level.INFO,"Verifying parallel execution of test methods for test named {0} with thread count {1}", + new Object[] {testName, maxSimultaneousTestMethods}); + + logger.log(Level.INFO, "{0} test method event logs for {1} test methods: ", + new Object[]{ testMethodEventLogs.size(), testMethodEventLogs.size()/3} ); + + //There are three test method events expected per test method: a start event, an execution event, and a test + //method pass event. All methods take exactly the same amount of time to execute. Each one of their events + //takes exactly the same time to execute. The reason for this is that it makes it possible to assume that blocks + //of methods should execute in parallel in lockstep, starting and finishing at the same time. + // + //The TestNgRunStateListener logs the start and pass events. The test method execution event is logged by the + //test method itself. See the sample test classes for examples. This test method verifies the parallel + //execution of test methods for parallelization tests involving parallel-by-methods mode. Therefore, the + //expectation is that there are simultaneously executing blocks of methods. The'size' of the block is either + //equal to the thread count or less in the event that the total number of methods is not a multiple of the + //thread count and we are processing the final block of methods to execute. + // + //This smaller, last block size is calculated using the number of events logged, the number of events logged + // per method (3) and the thread count to find the number of events expected for that remainder block. + int remainder = (testMethodEventLogs.size() / 3) % maxSimultaneousTestMethods; + + int numBlocks = testMethodEventLogs.size() / 3 < maxSimultaneousTestMethods ? 1 : + (testMethodEventLogs.size() / 3) / maxSimultaneousTestMethods + (remainder > 0 ? 1 : 0); + + log(testMethodEventLogs.size(), maxSimultaneousTestMethods, remainder); + + int loopNum = 1; + + //Loop over the event logs. The increment is equal the thread count times the number of events logged for each + //test method. for (int i = 1; i < testMethodEventLogs.size(); i = i + maxSimultaneousTestMethods * 3) { + + logger.log(Level.INFO, "Processing block {0} of {1}", new Object[] {loopNum, numBlocks}); + + //The size of the block is equal to the thread count or the number of methods left over in the last block + //if the total number of methods is not a multiple of the thread count. Example: For a test run with 19 + //methods in total and a thread count of 7, the remainder is 5 methods. The last block of methods to execute + //would have 5 methods executing simultaneously. If the remainder is non-zero, and we are processing the + //last block of methods then the block size is less than the thread count. Otherwise, the block size is + //equal to the thread count. int blockSize = (remainder != 0 && testMethodEventLogs.size() - i < maxSimultaneousTestMethods * 3) ? - remainder / 3 : + remainder : maxSimultaneousTestMethods; + //The expectation for the block of methods executing in parallel is that the test method start events are + //logged first, then the test method execution events, followed by the test method pass events. These + //offset values are used to extract the sublists of start events, execution events and pass events for the + //block of event logs to process in the current loop execution. int offsetOne = (remainder != 0 && testMethodEventLogs.size() - i < maxSimultaneousTestMethods * 3) ? - testMethodEventLogs.size() - remainder : + testMethodEventLogs.size() - (remainder * 3) : i - 1; int offsetTwo = (remainder != 0 && testMethodEventLogs.size() - i < maxSimultaneousTestMethods * 3) ? - testMethodEventLogs.size() - remainder + blockSize : + testMethodEventLogs.size() - (remainder * 3) + blockSize : i + maxSimultaneousTestMethods - 1; + logger.log(Level.INFO, "Expecting {0} test method start events, followed by {0} test method execution " + + "events, followed by {0} test method pass events", blockSize); + List eventLogMethodListenerStartSublist = testMethodEventLogs.subList(offsetOne, offsetTwo); List eventLogMethodExecuteSublist = testMethodEventLogs.subList(offsetTwo, offsetTwo + blockSize); List eventLogMethodListenerPassSublist = testMethodEventLogs.subList(offsetTwo + blockSize, offsetTwo + 2 * blockSize); + log(offsetOne, offsetTwo, blockSize, eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, + eventLogMethodListenerPassSublist); + + //Verify that all the events in the sublist extracted for the start events of the block of methods expected + //to execute in parallel all have the test method start event type and that they all executed in different + //threads. verifySimultaneousTestMethodListenerStartEvents(eventLogMethodListenerStartSublist, testName, - maxSimultaneousTestMethods); + blockSize); + //Verify that all the events in the sublist extracted for the test method execution events of the block of + //methods expected to execute in parallel all have the test method execution event type and that they all + //executed in different threads. verifySimultaneousTestMethodExecutionEvents(eventLogMethodExecuteSublist, testName, - maxSimultaneousTestMethods); + blockSize); + //Verify that the test method start events and the test method execution events in the two sublists belong + //to the same methods. This is done by verifying that the test class names and method names are the same for + //for both sublists. verifyEventsBelongToSameMethods(eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, "The " + "expected maximum number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + maxSimultaneousTestMethods + " methods should be " + @@ -342,42 +410,121 @@ public static void verifySimultaneousTestMethods(List testMethodEventL "test methods should all belong to the same methods as the test method listener onTestStart " + "event logs immediately preceding"); + //Verify that all the events in the sublist extracted for the test method pass events of the block of + //methods expected to execute in parallel all have the test method pass event type and that they all + //executed in different threads. verifySimultaneousTestMethodListenerPassEvents(eventLogMethodListenerPassSublist, testName, - maxSimultaneousTestMethods); + blockSize); + //Verify that the test method execution events and the test method pass events in the two sublists belong + //to the same methods. This is done by verifying that the test class names and method names are the same for + //for both sublists. verifyEventsBelongToSameMethods(eventLogMethodExecuteSublist, eventLogMethodListenerPassSublist, "The " + "expected maximum number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + maxSimultaneousTestMethods + " methods should be " + "running at the same time. The test method listener on onTestSuccess event logs for a block of " + "simultaneously running test methods should all belong to the same methods as the test method " + "execution event logs immediately preceding"); + + loopNum++; } } + //Verify that methods associated with the specified event logs execute simultaneously in parallel fashion, in + //accordance with the thread count. This verification is for blocks of parallel methods, some of which use + //non-parallel data providers without factories, so all the invocations of the methods run on the same class + //instances. This verification is for blocks of parallel methods that have the same sleep delays for their + //execution bodies and which do not have any BeforeMethod AfterMethod, BeforeGroup or AfterGroup + //configuration methods. public static void verifyParallelTestMethodsWithNonParallelDataProvider(List testMethodEventLogs, String testName, Map expectedInvocationCounts, int numUniqueMethods, int maxSimultaneousTestMethods) { + logger.log(Level.INFO,"Verifying parallel execution of test methods using non-parallel data providers for " + + "test named {0} with thread count {1}", new Object[] {testName, maxSimultaneousTestMethods}); + + logger.log(Level.INFO, "{0} test method event logs for {1} unique methods: ", + new Object[] {testMethodEventLogs.size(), numUniqueMethods}); + + //Some of the test methods use non-parallel data providers without factories. All the invocations of those + //test methods will occur serially within the same thread on the same class instances. In order to ensure that + //the loop logic below works properly, it is necessary to keep state information about which methods are + //supposed to be executing within a block of parallel methods that are running simultaneously. Unlike the + //logic in verifySimultaneousTestMethods, it is not possible to assume that methods within a block of + //simultaneously executing methods start and finish at the same time because the methods will frequently be + //invoked a varying number of times, depending on their use of data providers. + // + //However, each _invocation_ of a test method should take exactly the same amount of time. There are three test + //method events expected per test method: a start event, an execution event, and a test method pass event. All + //test method events of the same time take the same amount of time to execute. The reason for this is that it + //makes it possible to assume that blocks of method invocations should execute in parallel in lockstep, + //starting and finishing at the same time. Map methodsExecuting = new HashMap<>(); + + //This isn't actually used for any verification logic. I may remove it in the future. Map methodsCompleted = new HashMap<>(); + + //Because this method verifies combination of parallel-by-methods mode and the use of non-parallel data + //providers without factories, it is necessary to track the number of times that test methods are invoked in + //order to check that this is consistent with the number of times they are expected to be invoked based on + //their use of non-parallel data providers. Map methodInvocationsCounts = new HashMap<>(); + + //In order to verify that all invocations of test methods which use non-parallel data providers occur in the + //same thread, it is necessary to keep track of the thread IDs of methods that are executing within a + //block of simultaneously executing methods. Map executingMethodThreadIds = new HashMap<>(); + //The logic for determining the block size of simultaneously executing parallel methods is initially determined + //by whether the total number of unique methods less than the thread count. If it is less than the thread count, + //then the block size is equal to the number of unique methods. Those methods will execute in parallel + //until all invocations of all the methods completes. Otherwise, there are more methods queued up than the + //thread count, so the block size is equal to the thread count. int blockSize = numUniqueMethods >= maxSimultaneousTestMethods ? maxSimultaneousTestMethods : numUniqueMethods; + int loopNum = 1; + for (int i = 1; i < testMethodEventLogs.size(); i = i + blockSize * 3) { + logger.log(Level.INFO, "Processing block {0}", loopNum); + + //If the loop is executing more than once, then the block size needs to be updated. The number of remaining + //unique methods to execute determines the block size of parallel methods expected to execute + //simultaneously. if(i != 1) { + + //All methods that are in the list of currently executing methods should still have invocations left. + //Otherwise, they would have been removed from that list and added to the completed methods list. + allExecutingMethodsHaveMoreInvocations(methodsExecuting, methodInvocationsCounts, + expectedInvocationCounts); + + //If there are no remaining unique methods, the block size of methods expected to be executing in + //parallel is equal to the number of methods using data providers that are already executing. The only + //test method event logs to verify in this loop should belong to test methods which use data providers + //and are executed multiple times as a result. if(numUniqueMethods == 0) { blockSize = methodsExecuting.keySet().size(); - } else if(numUniqueMethods < maxSimultaneousTestMethods) { - if(methodsExecuting.keySet().size() == maxSimultaneousTestMethods && - allExecutingMethodsHaveMoreInvocations(methodsExecuting, methodInvocationsCounts, - expectedInvocationCounts)) { - blockSize = methodsExecuting.keySet().size(); + } else { + //Otherwise, if the number of unique methods left is non-zero, but less than the thread count, + //the block size is dependent on whether the number of currently executing methods is equal to the + //thread count. If so, then the block size is equal to the thread count and no new unique methods + //will begin executing this block of methods. If the number of methods already executing is less + //than the thread count and the sum of the number of unique methods left and the number of currently + //executing methods is equal to or greater than the thread count, the block size is equal to the + //thread count. If the sum is less than the thread count, the block size is equal to the sum. + if(numUniqueMethods < maxSimultaneousTestMethods) { + + if (methodsExecuting.keySet().size() == maxSimultaneousTestMethods || + numUniqueMethods + methodsExecuting.keySet().size() >= maxSimultaneousTestMethods) { + blockSize = maxSimultaneousTestMethods; + } else { + blockSize = numUniqueMethods + methodsExecuting.keySet().size(); + } + //If the number of unique methods left is more than or equal to the thread count, the block size + //is equal to the thread count. } else { - blockSize = numUniqueMethods + methodsExecuting.keySet().size(); + blockSize = maxSimultaneousTestMethods; } } } @@ -386,21 +533,39 @@ public static void verifyParallelTestMethodsWithNonParallelDataProvider(List eventLogMethodListenerStartSublist = testMethodEventLogs.subList(offsetOne, offsetTwo); List eventLogMethodExecuteSublist = testMethodEventLogs.subList(offsetTwo, offsetTwo + blockSize); List eventLogMethodListenerPassSublist = testMethodEventLogs.subList(offsetTwo + blockSize, offsetTwo + 2 * blockSize); + log(offsetOne, offsetTwo, blockSize, eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, + eventLogMethodListenerPassSublist); + + //Verify that all the events in the sublist extracted for the start events of the block of methods expected + //to execute in parallel all have the test method start event type and that they all executed in different + //threads. The method should return the total number of new unique methods that began executing in the + //current block of parallel methods executing in parallel. int decrementUniqueMethods = verifySimultaneousTestMethodListenerStartEvents( - eventLogMethodListenerStartSublist, testName, maxSimultaneousTestMethods, methodsExecuting, + eventLogMethodListenerStartSublist, testName, blockSize, methodsExecuting, executingMethodThreadIds, methodInvocationsCounts, expectedInvocationCounts ); + //Decrement the unique of unique methods left to execute by the number of new unique methods that began + //execution in this block of parallel methods. numUniqueMethods = numUniqueMethods - decrementUniqueMethods; + //Verify that all the events in the sublist extracted for the test method execution events of the block of + //methods expected to execute in parallel all have the test method execution event type and that they all + //executed in different threads. verifySimultaneousTestMethodExecutionEvents(eventLogMethodExecuteSublist, testName, - executingMethodThreadIds, maxSimultaneousTestMethods); + executingMethodThreadIds, blockSize); + //Verify that the test method start events and the test method execution events in the two sublists belong + //to the same methods. This is done by verifying that the test class names and method names are the same for + //for both sublists. verifyEventsBelongToSameMethods(eventLogMethodListenerStartSublist, eventLogMethodExecuteSublist, "The " + "expected maximum number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + maxSimultaneousTestMethods + " methods should be " + @@ -408,10 +573,22 @@ public static void verifyParallelTestMethodsWithNonParallelDataProvider(List listenerStartEventLogs, String - testName, int maxSimultaneousTestMethods, Map methodsExecuting, Map + testName, int blockSize, Map methodsExecuting, Map executingMethodThreadIds, Map methodInvocationsCounts, Map expectedInvocationCounts) { - verifySimultaneousTestMethodListenerStartEvents(listenerStartEventLogs, testName, maxSimultaneousTestMethods); + verifySimultaneousTestMethodListenerStartEvents(listenerStartEventLogs, testName, blockSize); int decrement = 0; @@ -453,10 +631,12 @@ public static int verifySimultaneousTestMethodListenerStartEvents(List if (methodsExecuting.keySet().contains(classAndMethodName)) { assertTrue(eventLog.getThreadId() == executingMethodThreadIds.get(classAndMethodName), "All " + "invocations of method '" + classAndMethodName + "' should execute in the same " + - "thread"); + "thread, but some event logs have different thread IDs"); } else { assertFalse(executingMethodThreadIds.values().contains(eventLog.getThreadId()), "Event logs " + - "for currently executing methods should have different thread IDs: " + classAndMethodName); + "for different methods currently executing should have different thread IDs, but some event " + + "logs for different methods in the current block being processed have the same thread ID: " + + classAndMethodName); } if(methodsExecuting.get(classAndMethodName) == null) { @@ -469,33 +649,37 @@ public static int verifySimultaneousTestMethodListenerStartEvents(List } public static void verifySimultaneousTestMethodExecutionEvents(List testMethodExecutionEventLogs, String - testName, Map executingMethodThreadIds, int maxSimultaneousTestMethods) { + testName, Map executingMethodThreadIds, int blockSize) { - verifySimultaneousTestMethodExecutionEvents(testMethodExecutionEventLogs, testName, maxSimultaneousTestMethods); + verifySimultaneousTestMethodExecutionEvents(testMethodExecutionEventLogs, testName, blockSize); for(EventLog eventLog : testMethodExecutionEventLogs) { String classAndMethodName = (String)eventLog.getData(CLASS_NAME) + "." + (String)eventLog.getData(METHOD_NAME); assertTrue(eventLog.getThreadId() == executingMethodThreadIds.get(classAndMethodName), "All the " + - "test method event logs for a given method should have the same thread ID"); + "test method event logs for a given method using a non-parallel data provider should have the " + + "same thread ID, but some event logs for a method have different thread IDs: " + + classAndMethodName); } } public static void verifySimultaneousTestMethodListenerPassEvents(List testMethodListenerPassEventLogs, - String testName, int maxSimultaneousTestMethods, Map methodsExecuting, + String testName, int blockSize, Map methodsExecuting, Map methodsCompleted, Map executingMethodThreadIds, Map methodInvocationsCounts, Map expectedInvocationCounts) { verifySimultaneousTestMethodListenerPassEvents(testMethodListenerPassEventLogs, testName, - maxSimultaneousTestMethods); + blockSize); for(EventLog eventLog : testMethodListenerPassEventLogs) { String classAndMethodName = (String)eventLog.getData(CLASS_NAME) + "." + (String)eventLog.getData(METHOD_NAME); assertTrue(eventLog.getThreadId() == executingMethodThreadIds.get(classAndMethodName), "All the " + - "test method event logs for a given method should have the same thread ID"); + "test method event logs for a given method using a non-parallel data provider should have the " + + "same thread ID, but some event logs for a method have different thread IDs: " + + classAndMethodName); if(methodInvocationsCounts.get(classAndMethodName) .equals(expectedInvocationCounts.get(classAndMethodName))) { @@ -511,48 +695,45 @@ public static void verifySimultaneousTestMethodListenerPassEvents(List //according to the expected maximum number of simultaneous executions. Verifies that each of them has the same //event type and all have different thread IDs. public static void verifySimultaneousTestMethodListenerStartEvents(List listenerStartEventLogs, String - testName, int maxSimultaneousTestMethods) { + testName, int blockSize) { + + verifyEventTypeForEventsLogs(listenerStartEventLogs, LISTENER_TEST_METHOD_START, "Expected " + blockSize + + " test method start event logs to be in a block of methods executing in parallel. Found an event log " + + "of a different type in the block being processed: " + listenerStartEventLogs); - verifyEventTypeForEventsLogs(listenerStartEventLogs, LISTENER_TEST_METHOD_START, "The expected maximum " + - "number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + - " so more no more than " + maxSimultaneousTestMethods + " methods should start running at the same " + - "time if there are more than " + maxSimultaneousTestMethods + " methods remaining to execute. Event " + - "logs: " + listenerStartEventLogs); - verifyDifferentThreadIdsForEvents(listenerStartEventLogs, "The expected maximum number of methods to execute " + - "simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so the thread IDs for all " + - "the test method listener's onTestStart method " + "the " + maxSimultaneousTestMethods + "currently " + - "executing test methods should be different. Event logs: " + listenerStartEventLogs); + verifyDifferentThreadIdsForEvents(listenerStartEventLogs, "Expected " + blockSize + " test method start " + + "event logs to be in a block of methods executing in parallel. Each one of these event logs should " + + "be associated with a different thread ID, but found that at two event logs share the same thread " + + "ID: " + listenerStartEventLogs); } //Verify that the specified test method execution event logs execute simultaneously in parallel fashion according //to the specified thread count. Verifies that each of them has the same event type and all have different thread //IDs. public static void verifySimultaneousTestMethodExecutionEvents(List testMethodExecutionEventLogs, - String testName, int maxSimultaneousTestMethods) { - - verifyEventTypeForEventsLogs(testMethodExecutionEventLogs, TEST_METHOD_EXECUTION, "The expected maximum " + - "number of methods to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + - " so no more than " + maxSimultaneousTestMethods + " methods should be " + "executing at the same " + - "time. Event logs: " + testMethodExecutionEventLogs); - verifyDifferentThreadIdsForEvents(testMethodExecutionEventLogs, "The expected maximum number of methods to " + - "execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so the thread IDs " + - "for the test method execution events for the " + maxSimultaneousTestMethods + "currently executing " + - "test methods should be different. Event logs: " + testMethodExecutionEventLogs); + String testName, int blockSize) { + + verifyEventTypeForEventsLogs(testMethodExecutionEventLogs, TEST_METHOD_EXECUTION, "Expected " + blockSize + + " test method execution event logs to be in a block of methods executing in parallel. Found an event " + + "log of a different type in the block being processed: " + testMethodExecutionEventLogs); + verifyDifferentThreadIdsForEvents(testMethodExecutionEventLogs, "Expected " + blockSize + " test method " + + "execution event logs to be in a block of methods executing in parallel. Each one of these event " + + "logs should be associated with a different thread ID, but found that at two event logs share the " + + "same thread ID: " + testMethodExecutionEventLogs); } //Verify that the specified test method listener onTestSuccess event logs execute simultaneously in parallel //fashion according to the specified thread count. Verifies that each of them has the same event type and all have //different thread IDs. public static void verifySimultaneousTestMethodListenerPassEvents(List testMethodListenerPassEventLogs, - String testName, int maxSimultaneousTestMethods) { - verifyEventTypeForEventsLogs(testMethodListenerPassEventLogs, LISTENER_TEST_METHOD_PASS, "The thread " + - "count is " + maxSimultaneousTestMethods + " for " + testName + " so no more than " + - maxSimultaneousTestMethods + " test listener " + "onTestSuccess methods should be executing at the " + - "same time. Event logs: " + testMethodListenerPassEventLogs); - verifyDifferentThreadIdsForEvents(testMethodListenerPassEventLogs, "The expected maximum number of methods " + - "to execute simultaneously is " + maxSimultaneousTestMethods + " for " + testName + " so the thread " + - "IDs for the test method listener onTestSuccess events for the " + maxSimultaneousTestMethods + - "currently executing test methods should be different. Event logs: " + testMethodListenerPassEventLogs); + String testName, int blockSize) { + verifyEventTypeForEventsLogs(testMethodListenerPassEventLogs, LISTENER_TEST_METHOD_PASS, "Expected " + + blockSize + " test method pass event logs to be in a block of methods executing in parallel. Found " + + "an event log of a different type in the block being processed: " + testMethodListenerPassEventLogs); + verifyDifferentThreadIdsForEvents(testMethodListenerPassEventLogs, "Expected " + blockSize + " test method " + + "pass event logs to be in a block of methods executing in parallel. Each one of these event " + + "logs should be associated with a different thread ID, but found that at two event logs share the " + + "same thread ID: " + testMethodListenerPassEventLogs); } //Verify that the test method level events for the test methods declared in the specified class run in the same @@ -627,7 +808,7 @@ public static void verifySequentialSuites(List suiteLevelEventLogs, Ma assertTrue(suiteLevelEventLogs.get(i).getEvent() == LISTENER_SUITE_START && suiteLevelEventLogs.get(i + 1).getEvent() == LISTENER_SUITE_FINISH, "Because the suites are " + "expected to execute sequentially, the suite level event logs should consist of a series of " + - "pairs of a suite listener onStart event log followed by a suite listener onFinish event log: " + + "pairs of a suite listener onStart event logger followed by a suite listener onFinish event logger: " + suiteLevelEventLogs); suiteListenerStartEventLogs.add((suiteLevelEventLogs.get(i))); } @@ -640,7 +821,7 @@ public static void verifySequentialSuites(List suiteLevelEventLogs, Ma List secondSuiteEventLogs = suiteEventLogsMap.get(secondSuite); verifySequentialTimingOfEvents(firstSuiteEventLogs, secondSuiteEventLogs, "The first suite listener " + - "onStart event log is for " + firstSuite + " and the second suite listener onStart event log is " + + "onStart event logger is for " + firstSuite + " and the second suite listener onStart event logger is " + "for " + secondSuite + ". Because the suites are supposed to execute sequentially, all of the " + "event logs for " + firstSuite + " should have timestamps earlier than all of the event logs for " + secondSuite + ". First suite event logs: " + firstSuiteEventLogs + ". Second suite event logs: " + @@ -661,15 +842,15 @@ public static void verifySequentialTests(List suiteAndTestLevelEventLo verifyEventsOccurBetween(suiteListenerOnStartEventLog, testLevelEventLogs, suiteListenerOnFinishEventLog, "All of the test level event logs should have timestamps between the suite listener's onStart and " + - "onFinish event logs. Suite listener onStart event log: " + suiteListenerOnStartEventLog + - ". Suite listener onFinish event log: " + suiteListenerOnFinishEventLog + ". Test level " + + "onFinish event logs. Suite listener onStart event logger: " + suiteListenerOnStartEventLog + + ". Suite listener onFinish event logger: " + suiteListenerOnFinishEventLog + ". Test level " + "event logs: " + testLevelEventLogs); for (int i = 0; i < testLevelEventLogs.size(); i = i + 2) { assertTrue(testLevelEventLogs.get(i).getEvent() == LISTENER_TEST_START && testLevelEventLogs.get(i + 1).getEvent() == LISTENER_TEST_FINISH, "Because the tests are " + "expected to execute sequentially, the test level event logs should consist of a series of " + - "pairs of a test listener onStart event log followed by a test listener onFinish event log: " + + "pairs of a test listener onStart event logger followed by a test listener onFinish event logger: " + testLevelEventLogs); } } @@ -733,8 +914,8 @@ public static void verifyParallelSuitesWithUnequalExecutionTimes(List if (eventLog.getEvent() == LISTENER_SUITE_FINISH) { - assertTrue(suitesExecuting.get(suiteName) != null, "Found an event log for a suite listener " + - "onFinish event that does not have a corresponding event log for a suite listener " + + assertTrue(suitesExecuting.get(suiteName) != null, "Found an event logger for a suite listener " + + "onFinish event that does not have a corresponding event logger for a suite listener " + "onStart event"); assertTrue(suitesExecuting.get(suiteName).getThreadId() == eventLog.getThreadId(), "All the " + "suite level event logs for a given suite should have the same thread ID"); @@ -821,5 +1002,69 @@ private static boolean allExecutingMethodsHaveMoreInvocations(Map eventLogMethodListenerStartSublist, + List eventLogMethodExecuteSublist, List eventLogMethodListenerPassSublist) { + logger.log(Level.INFO, "Event logs extracted from event log list between index {0} and index {1} should " + + "be the test method start event logs for a block of {2} simultaneously executing methods", + new Object[] {offsetOne, offsetTwo - 1, blockSize}); + + int j = offsetOne; + + for(EventLog eventLog : eventLogMethodListenerStartSublist) { + logger.log(Level.INFO, "Event logged at index {0}: {1}", new Object[] {j, eventLog.toString()}); + j++; + } + + logger.log(Level.INFO, "Event logs extracted from event log list between index {0} and index {1} should " + + "be the test method execution event logs for a block of {2} simultaneously executing methods", + new Object[] {offsetTwo, offsetTwo + blockSize - 1, blockSize}); + + j = offsetTwo; + + for(EventLog eventLog : eventLogMethodExecuteSublist) { + logger.log(Level.INFO, "Event logged at index {0}: {1}", new Object[] {j, eventLog.toString()}); + j++; + } + + logger.log(Level.INFO, "Event logs extracted from event log list between index {0} and index {1} should " + + "be the test method pass event logs for a block of {2} simultaneously executing methods", + new Object[] {offsetTwo + blockSize, offsetTwo + 2 * blockSize - 1, blockSize}); + + j = offsetTwo + blockSize; + + for(EventLog eventLog : eventLogMethodListenerPassSublist) { + logger.log(Level.INFO, "Event logged at index {0}: {1}", new Object[] {j, eventLog.toString()}); + j++; + } + } + + private static void log(int listSize, int threadCount, int remainder) { + if(listSize / 3 < threadCount) { + logger.log(Level.INFO, "Expecting there to be a single block of {0} parallel methods", listSize / 3); + } else { + + if(remainder > 0) { + logger.log(Level.INFO, "Expecting there to be a series of {0} blocks of {1} parallel methods with a " + + "final block of {2} parallel methods", + + new Object[] + { + (listSize / 3) / threadCount, + threadCount, + remainder + } + ); + } else { + logger.log(Level.INFO, "Expecting there to be a series of {0} blocks of {1} parallel methods", + new Object[] + { + (listSize / 3) / threadCount, + threadCount + } + ); + } + } + } } diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java index 15f6809d4c..0549c6fe4c 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario1.java @@ -1,17 +1,19 @@ package test.thread.parallelization; -import com.google.common.collect.Multimap; import org.testng.ITestNGListener; import org.testng.TestNG; + import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; + import org.testng.xml.XmlSuite; import test.thread.parallelization.TestNgRunStateTracker.EventLog; import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; -import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -46,6 +48,12 @@ */ public class ParallelByMethodsTestCase1Scenario1 extends BaseParallelizationTest { + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -77,6 +85,13 @@ public void setUp() { TestNG tng = create(suite); tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario1. This test scenario consists of a " + + "single suite with a single test which consists of one test class with five test methods. There " + + "are no dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName(), 5}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java index 8ec9a35cba..84183f780b 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase1Scenario2.java @@ -11,6 +11,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import test.thread.parallelization.TestNgRunStateTracker.EventLog; @@ -77,6 +79,13 @@ * 10) There are no method exclusions */ public class ParallelByMethodsTestCase1Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase1Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -208,6 +217,46 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo, suiteThree); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase1Scenario2. This test scenario consists of three " + + "sequentially executed suites with 1, 2 and 3 tests respectively. One test for a suite consists of a " + + "single test class while the rest shall consist of more than one test class. There are no " + + "dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java index b8e81179b9..191dc9447a 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario1.java @@ -2,8 +2,10 @@ import org.testng.ITestNGListener; import org.testng.TestNG; + import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; + import org.testng.xml.XmlSuite; import test.thread.parallelization.TestNgRunStateTracker.EventLog; @@ -19,6 +21,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -62,6 +66,12 @@ */ public class ParallelByMethodsTestCase2Scenario1 extends BaseParallelizationTest { + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -125,7 +135,6 @@ public void setUp() { suiteTwo.setParallel(XmlSuite.ParallelMode.METHODS); suiteTwo.setThreadCount(14); - addParams(suiteOne, SUITE_A, SUITE_A_TEST_A, "100"); addParams(suiteTwo, SUITE_B, SUITE_B_TEST_A, "100"); @@ -135,6 +144,25 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario1. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. The suites run in parallel and the thread pool size is 2. " + + "One test for a suite shall consist of a single test class while the rest shall consist of more than " + + "one test class. There are no dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 14}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 14}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java index 17374fb298..4165205403 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase2Scenario2.java @@ -26,6 +26,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -71,6 +73,13 @@ * 11) There are no method exclusions */ public class ParallelByMethodsTestCase2Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase2Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -204,6 +213,46 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase2Scenario2. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is 3 " + + "One test for a suite shall consist of a single test class while the rest shall consist of more than " + + "one test class. There are no dependencies, data providers or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 6}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java index ab15cd6a99..ff94dc8fab 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario1.java @@ -10,8 +10,13 @@ import test.thread.parallelization.TestNgRunStateTracker.EventLog; import test.thread.parallelization.sample.TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample; +import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -25,6 +30,7 @@ import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartEventLog; import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartThreadId; +import static test.thread.parallelization.TestNgRunStateTracker.getTestMethodLevelEventLogsForTest; import static test.thread.parallelization.TestNgRunStateTracker.reset; /** @@ -48,6 +54,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase3Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -64,6 +77,8 @@ public class ParallelByMethodsTestCase3Scenario1 extends BaseParallelizationTest private Long testListenerOnStartThreadId; + private Map expectedInvocationCounts = new HashMap<>(); + @BeforeClass public void setUp() { reset(); @@ -72,6 +87,26 @@ public void setUp() { suite.setParallel(XmlSuite.ParallelMode.METHODS); suite.setThreadCount(15); + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodA", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodB", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodC", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodD", 3); + + expectedInvocationCounts.put( + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + + ".testMethodE", 3); + createXmlTest(suite, TEST, TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class); addParams(suite, SUITE, TEST, "100", "paramOne,paramTwo,paramThree"); @@ -80,6 +115,14 @@ public void setUp() { tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase3Scenario1. This test scenario consists of a " + + "single suite with a single test consisting of a single test class with five methods with a data " + + "provider specifying 3 sets of data. There are no dependencies or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST, + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName(), 15}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); @@ -168,7 +211,13 @@ public void verifyThatMethodLevelEventsRunInDifferentThreadsFromSuiteAndTestLeve //Verifies that the test methods execute in different threads in parallel fashion. @Test public void verifyThatTestMethodsRunInParallelThreads() { - verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5); + + verifyParallelTestMethodsWithNonParallelDataProvider( + getTestMethodLevelEventLogsForTest(SUITE, TEST), TEST, expectedInvocationCounts, + 5, 5 + ); + + //verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 5); } //Verifies that all the test method level events for any given test method run in the same thread. diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java index d71fc62e8a..41f18f67b2 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase3Scenario2.java @@ -17,6 +17,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -60,6 +62,12 @@ */ public class ParallelByMethodsTestCase3Scenario2 extends BaseParallelizationTest { + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase3Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -123,8 +131,8 @@ public void setUp() { TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class); suiteOne.setParallel(XmlSuite.ParallelMode.METHODS); - suiteOne.setThreadCount(3); + suiteTwo.setParallel(XmlSuite.ParallelMode.METHODS); suiteTwo.setThreadCount(4); @@ -137,6 +145,27 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase3Scenario2. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. One test for a suite shall consist of a single test class " + + "while the rest shall consist of more than one test class. Each test class has some methods with use " + + "a data provider and some which do not. Two data providers are used: one which provides two sets of " + + "data, one which provide three sets of data. There are no dependencies or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassBSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A, + TestClassCFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 4}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassEFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 4}); + tng.run(); expectedInvocationCounts.put( diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java index 0d346266e2..028e486cbe 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase4Scenario1.java @@ -27,6 +27,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllEventLogsForSuite; @@ -78,6 +80,13 @@ * 11) There are no method exclusions */ public class ParallelByMethodsTestCase4Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase4Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -224,6 +233,50 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase4Scenario1. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is " + + "2. One test for a suite shall consist of a single test class while the rest shall consist of more " + + "han one test class. Some test classes have a mixture of some methods that use a data provider and " + + "some which do not. The data providers provide data sets of varying sizes. There are no " + + "dependencies or factories."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassBSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A, + TestClassCFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName(), 6}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class + ", " + + TestClassEFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class + ", " + + TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassDThreeMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassGFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassAFiveMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassBFourMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassIThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassJFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample.class.getCanonicalName() + ", " + + TestClassBSixMethodsWithDataProviderOnAllMethodsAndNoDepsSample.class.getCanonicalName(), + 12}); + tng.run(); expectedInvocationCounts.put( diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java index 7c5fc07422..2c00a433cc 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario1.java @@ -14,6 +14,8 @@ import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteAndTestLevelEventLogs; @@ -48,6 +50,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase5Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase5Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -80,6 +89,13 @@ public void setUp() { tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase5Scenario1. This test scenario consists of a " + + "single suite with a single test consisting of a factory that provides two instances of a single\n" + + "test class with five methods. There are no dependencies or data providers."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST, TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName(), 15}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java index 17539f7ab1..c3433a4566 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase5Scenario2.java @@ -13,17 +13,25 @@ import test.thread.parallelization.sample.FactoryForTestClassDThreeMethodsWithNoDepsFourInstancesSample; import test.thread.parallelization.sample.FactoryForTestClassFSixMethodsWithNoDepsSixInstancesSample; +import test.thread.parallelization.sample.TestClassAFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassBFourMethodsWithNoDepsSample; +import test.thread.parallelization.sample.TestClassBSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; +import test.thread.parallelization.sample.TestClassCFiveMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassCSixMethodsWithNoDepsSample; +import test.thread.parallelization.sample.TestClassDThreeMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassDThreeMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassEFiveMethodsWithNoDepsSample; +import test.thread.parallelization.sample.TestClassEFourMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; +import test.thread.parallelization.sample.TestClassFSixMethodsWithDataProviderOnSomeMethodsAndNoDepsSample; import test.thread.parallelization.sample.TestClassFSixMethodsWithNoDepsSample; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllEventLogsForSuite; @@ -66,6 +74,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase5Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase5Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -145,6 +160,26 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase5Scenario2. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. One suite with two tests has a test consisting of a single " + + "test class without a factory while the other consists of factories which provide multiple instances " + + "of multiple test classes. One suite shall consist of a single test with multiple test classes which " + + "uses factories. There are no dependencies or data providers."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java index fd97149c8a..32be90d6cb 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase6Scenario1.java @@ -31,6 +31,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteLevelEventLogs; @@ -80,6 +82,13 @@ * 12) There are no method exclusions */ public class ParallelByMethodsTestCase6Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase6Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -214,6 +223,48 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase6Scenario1. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. The suites run in parallel and the thread pool size is " + + "2. One suite with two tests has a test consisting of a single test class without a factory while " + + "the other shall consist of factories which provide multiple instances of multiple test classes. One " + + "suite shall consist of a single test with multiple test classes which uses factories. One suite " + + "shall have multiple tests with multiple classes, none of which use a factory. There are no " + + "dependencies."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A,TestClassAFiveMethodsWithNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithNoDepsSample.class.getCanonicalName(), 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithNoDepsSample.class + ", " + + TestClassBFourMethodsWithNoDepsSample.class + ", " + + TestClassFSixMethodsWithNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java index 37e159b9ce..1db213cac4 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario1.java @@ -12,8 +12,11 @@ import test.thread.parallelization.sample.TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; + import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteAndTestLevelEventLogs; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteLevelEventLogs; import static test.thread.parallelization.TestNgRunStateTracker.getAllTestLevelEventLogs; @@ -24,6 +27,7 @@ import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartEventLog; import static test.thread.parallelization.TestNgRunStateTracker.getTestListenerStartThreadId; +import static test.thread.parallelization.TestNgRunStateTracker.getTestMethodLevelEventLogsForTest; import static test.thread.parallelization.TestNgRunStateTracker.reset; /** This class covers PTP_TC_7, Scenario 1 in the Parallelization Test Plan. @@ -48,6 +52,13 @@ * 8) There are no method exclusions */ public class ParallelByMethodsTestCase7Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase7Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE = "SingleTestSuite"; private static final String TEST = "SingleTestClassTest"; @@ -80,6 +91,14 @@ public void setUp() { tng.addListener((ITestNGListener)new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase7Scenario1. This test scenario consists of a " + + "single suite with a single test consisting of a single test class with five methods with a " + + "factory method using a data provider specifying 3 sets of data. There are no dependencies."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE,TEST, + TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName(), 15}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); @@ -176,7 +195,6 @@ public void verifyThatAllEventsForATestMethodExecuteInSameThread() { //Verifies that the test methods execute in different threads in parallel fashion. @Test public void verifyThatTestMethodsRunInParallelThreads() { - verifySimultaneousTestMethods(testMethodLevelEventLogs, TEST, 15); + verifySimultaneousTestMethods(getTestMethodLevelEventLogsForTest(SUITE, TEST), TEST, 15); } - } diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java index 03e6981dcb..ddbe982554 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase7Scenario2.java @@ -8,9 +8,11 @@ import org.testng.xml.XmlTest; import test.thread.parallelization.sample.TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample; +import test.thread.parallelization.sample.TestClassAFiveMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassBFourMethodsWithFactoryUsingDataProviderAndNoDepsSample; import test.thread.parallelization.sample.TestClassBFourMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassCSixMethodsWithFactoryUsingDataProviderAndNoDepsSample; +import test.thread.parallelization.sample.TestClassCSixMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassDThreeMethodsWithFactoryUsingDataProviderAndNoDepsSample; import test.thread.parallelization.sample.TestClassDThreeMethodsWithNoDepsSample; import test.thread.parallelization.sample.TestClassEFiveMethodsWithNoDepsSample; @@ -21,6 +23,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; @@ -65,6 +69,13 @@ * 7) There are no method exclusions */ public class ParallelByMethodsTestCase7Scenario2 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase7Scenario2.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; @@ -157,6 +168,28 @@ public void setUp() { TestNG tng = create(suiteOne, suiteTwo); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase7Scenario2. This test scenario consists of two " + + "suites with 1 and 2 tests respectively. One suite with two tests has a test consisting of a single " + + "test class without a factory while the other consists of factories using data providers with " + + "varying numbers of data sets which provide multiple instances of multiple test classes. One suite " + + "shall consist of a single test with multiple test classes which use factories with data providers " + + "with varying numbers of data sets. There are no dependencies."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName(), 25}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassBFourMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassFSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class, + 40}); + tng.run(); suiteLevelEventLogs = getAllSuiteLevelEventLogs(); diff --git a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java index b963d583b0..a92f458a1f 100644 --- a/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java +++ b/src/test/java/test/thread/parallelization/ParallelByMethodsTestCase8Scenario1.java @@ -35,6 +35,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import static org.testng.Assert.assertEquals; import static test.thread.parallelization.TestNgRunStateTracker.getAllSuiteLevelEventLogs; @@ -86,6 +88,13 @@ * 12) here are no method exclusions */ public class ParallelByMethodsTestCase8Scenario1 extends BaseParallelizationTest { + + private static final Logger logger = Logger.getLogger(ParallelByMethodsTestCase8Scenario1.class.getCanonicalName()); + + { + logger.setLevel(Level.INFO); + } + private static final String SUITE_A = "TestSuiteA"; private static final String SUITE_B = "TestSuiteB"; private static final String SUITE_C = "TestSuiteC"; @@ -227,6 +236,48 @@ public void setUp() { tng.setSuiteThreadPoolSize(2); tng.addListener((ITestNGListener) new TestNgRunStateListener()); + logger.log(Level.INFO, "Beginning ParallelByMethodsTestCase8Scenario1. This test scenario consists of three " + + "suites with 1, 2 and 3 tests respectively. One suite with two tests has a test consisting of a " + + "single test class without a factory while the other shall consist of test classes with factories " + + "using data providers with varying numbers of data sets. One suite shall consist of a single test " + + "with multiple test classes with factories using data providers with varying numbers of data sets. " + + "One suite shall have multiple tests with multiple classes, none of which use a factory."); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_A,SUITE_A_TEST_A, + TestClassAFiveMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName() + + ", " + TestClassCSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class.getCanonicalName(), 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test class: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_A,TestClassEFiveMethodsWithNoDepsSample.class.getCanonicalName(), 3}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_B,SUITE_B_TEST_B, + TestClassDThreeMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassBFourMethodsWithFactoryUsingDataProviderAndNoDepsSample.class + ", " + + TestClassFSixMethodsWithFactoryUsingDataProviderAndNoDepsSample.class, + 20}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_A, + TestClassGThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassHFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassIFiveMethodsWithNoDepsSample.class, + 10}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_B, + TestClassJFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassKFiveMethodsWithNoDepsSample.class, + 5}); + + logger.log(Level.INFO, "Suite: {0}, Test: {1}, Test classes: {2}. Thread count: {3}", + new Object[]{SUITE_C,SUITE_C_TEST_C, + TestClassLThreeMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassMFourMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassNFiveMethodsWithNoDepsSample.class.getCanonicalName() + ", " + + TestClassOSixMethodsWithNoDepsSample.class.getCanonicalName(), + 12}); tng.run(); expectedInvocationCounts.put( diff --git a/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java b/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java index 706424a2db..37c2a3c7af 100644 --- a/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java +++ b/src/test/java/test/thread/parallelization/TestNgRunStateTracker.java @@ -16,7 +16,7 @@ */ public class TestNgRunStateTracker { - private static List eventLogs = new ArrayList<>(); + private static final List eventLogs = new ArrayList<>(); public static void logEvent(EventLog eventLog) { synchronized(eventLogs) { @@ -754,7 +754,7 @@ public static Map getTestMethodExecutionThreadIds(String suiteName public static void reset() { - eventLogs = new ArrayList<>(); + eventLogs.clear(); } private static boolean isSuiteLevelEventLog(EventLog eventLog) { @@ -872,12 +872,35 @@ public static EventLogBuilder builder() { @Override public String toString() { - return "EventLog{" + - "event=" + event + - ", timeOfEvent=" + timeOfEvent + - ", threadId=" + threadId + - ", data=" + data + - '}'; + final StringBuffer sb = new StringBuffer("EventLog{"); + sb.append("Event: ").append(event); + + sb.append(", Suite: ").append(getData(EventInfo.SUITE_NAME)); + + if(getData(EventInfo.TEST_NAME) != null) { + sb.append(", Test: ").append(getData(EventInfo.TEST_NAME)); + } + + if(getData(EventInfo.CLASS_NAME) != null) { + sb.append(", Class: ").append(getData(EventInfo.CLASS_NAME)); + } + + if(getData(EventInfo.CLASS_INSTANCE) != null) { + sb.append(", Class instance hash code: ").append(getData(EventInfo.CLASS_INSTANCE).hashCode()); + } + + if(getData(EventInfo.METHOD_NAME) != null) { + sb.append(", Method name: ").append(getData(EventInfo.METHOD_NAME)); + } + + if(getData(EventInfo.DATA_PROVIDER_PARAM) != null) { + sb.append(", Data provider param: ").append(getData(EventInfo.DATA_PROVIDER_PARAM)); + } + + sb.append(", Time of event: ").append(timeOfEvent); + sb.append(", Thread ID: ").append(threadId); + sb.append("}"); + return sb.toString(); } }