Draft: Maestro Runner + Maestro Case Generation

agent/src/main/java/com/microsoft/hydralab/agent/config/TestRunnerConfig.java

@@ -8,6 +8,7 @@
 import com.microsoft.hydralab.agent.runner.appium.AppiumCrossRunner;
 import com.microsoft.hydralab.agent.runner.appium.AppiumRunner;
 import com.microsoft.hydralab.agent.runner.espresso.EspressoRunner;
+import com.microsoft.hydralab.agent.runner.maestro.MaestroRunner;
 import com.microsoft.hydralab.agent.runner.monkey.AdbMonkeyRunner;
 import com.microsoft.hydralab.agent.runner.monkey.AppiumMonkeyRunner;
 import com.microsoft.hydralab.agent.runner.smart.SmartRunner;
@@ -41,7 +42,8 @@ public class TestRunnerConfig {
             TestTask.TestRunningType.MONKEY_TEST, "adbMonkeyRunner",
             TestTask.TestRunningType.APPIUM_MONKEY_TEST, "appiumMonkeyRunner",
             TestTask.TestRunningType.T2C_JSON_TEST, "t2cRunner",
-            TestTask.TestRunningType.XCTEST, "xctestRunner"
+            TestTask.TestRunningType.XCTEST, "xctestRunner",
+            TestTask.TestRunningType.MAESTRO, "maestroRunner"
     );
 
     @Bean
@@ -125,6 +127,14 @@ public XCTestRunner xctestRunner(AgentManagementService agentManagementService,
         return new XCTestRunner(agentManagementService, testTaskEngineService, testRunDeviceOrchestrator, performanceTestManagementService);
     }
 
+    @Bean
+    public MaestroRunner maestroRunner(AgentManagementService agentManagementService,
+                                       TestTaskEngineService testTaskEngineService,
+                                       TestRunDeviceOrchestrator testRunDeviceOrchestrator,
+                                       PerformanceTestManagementService performanceTestManagementService) {
+        return new MaestroRunner(agentManagementService, testTaskEngineService, testRunDeviceOrchestrator, performanceTestManagementService);
+    }
+
     @ConfigurationProperties(prefix = "app.device-script.commands")
     @Bean(name = "DeviceCommandProperty")
     public List<DeviceScriptCommand> deviceCommandProperty() {

agent/src/main/java/com/microsoft/hydralab/agent/runner/XmlBuilder.java

@@ -31,7 +31,7 @@
  */
 public class XmlBuilder {
     private static final String TEST_RESULT_FILE_SUFFIX = ".xml";
-    private static final String TEST_RESULT_FILE_PREFIX = "hydra_result_";
+    public static final String TEST_RESULT_FILE_PREFIX = "hydra_result_";
     private static final String TEST_RESULT_FILE_ENCODE_PROPERTY = "encoding";
     private static final String TEST_RESULT_FILE_ENCODE_VALUE = "UTF-8";
     private static final String TEST_RESULT_FILE_OUTPUT_VALUE = "xml";
@@ -52,6 +52,14 @@ public class XmlBuilder {
     private static final String HOSTNAME = "hostname";
 
     public String buildTestResultXml(TestTask testTask, TestRun testRun) throws Exception {
+        File resultFolder = testRun.getResultFolder();
+        File[] files = resultFolder.listFiles();
+        for (File tempfile : files) {
+            if (tempfile.getName().startsWith(TEST_RESULT_FILE_PREFIX) && tempfile.getName().endsWith(TEST_RESULT_FILE_SUFFIX)) {
+                testRun.getLogger().info("find test result file: " + tempfile.getAbsolutePath());
+                return tempfile.getAbsolutePath();
+            }
+        }
         DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
         DocumentBuilder db = factory.newDocumentBuilder();
         Document document = db.newDocument();

agent/src/main/java/com/microsoft/hydralab/agent/runner/espresso/XmlTestRunListener.java

@@ -24,6 +24,7 @@
 import com.android.ddmlib.testrunner.TestResult.TestStatus;
 import com.android.ddmlib.testrunner.TestRunResult;
 import com.google.common.collect.ImmutableMap;
+import com.microsoft.hydralab.agent.runner.XmlBuilder;
 import com.microsoft.hydralab.common.util.FileUtil;
 import org.jetbrains.annotations.NotNull;
 import org.kxml2.io.KXmlSerializer;
@@ -53,7 +54,6 @@ public class XmlTestRunListener implements ITestRunListener {
     private static final String LOG_TAG = "XmlResultReporter";
 
     private static final String TEST_RESULT_FILE_SUFFIX = ".xml";
-    private static final String TEST_RESULT_FILE_PREFIX = "test_result_";
 
     private static final String TESTSUITE = "testsuite";
     private static final String TESTCASE = "testcase";
@@ -233,7 +233,7 @@ public void addSystemError(String systemError) {
      * @throws IOException
      */
     protected File getResultFile(File reportDir) throws IOException {
-        File reportFile = File.createTempFile(TEST_RESULT_FILE_PREFIX, TEST_RESULT_FILE_SUFFIX,
+        File reportFile = File.createTempFile(XmlBuilder.TEST_RESULT_FILE_PREFIX, TEST_RESULT_FILE_SUFFIX,
                 reportDir);
         Log.i(LOG_TAG, String.format("Created xml report file at %s",
                 reportFile.getAbsolutePath()));

agent/src/main/java/com/microsoft/hydralab/agent/runner/maestro/MaestroListener.java

@@ -0,0 +1,180 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package com.microsoft.hydralab.agent.runner.maestro;
+
+import com.microsoft.hydralab.agent.runner.TestRunDeviceOrchestrator;
+import com.microsoft.hydralab.common.entity.common.AndroidTestUnit;
+import com.microsoft.hydralab.common.entity.common.TestRun;
+import com.microsoft.hydralab.common.entity.common.TestRunDevice;
+import com.microsoft.hydralab.common.entity.common.TestTask;
+import com.microsoft.hydralab.common.management.AgentManagementService;
+import com.microsoft.hydralab.common.util.FileUtil;
+import com.microsoft.hydralab.performance.PerformanceTestListener;
+import org.slf4j.Logger;
+
+import java.io.File;
+
+/**
+ * @author zhoule
+ * @date 07/19/2023
+ */
+
+public class MaestroListener {
+    private final TestRunDevice testRunDevice;
+    private final TestRun testRun;
+    private final TestTask testTask;
+    private final Logger logger;
+    private final String pkgName;
+    private final AgentManagementService agentManagementService;
+    private final PerformanceTestListener performanceTestListener;
+    private TestRunDeviceOrchestrator testRunDeviceOrchestrator;
+    private long recordingStartTimeMillis;
+    private int index;
+    private boolean alreadyEnd = false;
+    private AndroidTestUnit ongoingTestUnit;
+    private int numTests;
+
+    public MaestroListener(AgentManagementService agentManagementService,
+                           TestRunDevice testRunDevice, TestRun testRun, TestTask testTask,
+                           TestRunDeviceOrchestrator testRunDeviceOrchestrator,
+                           PerformanceTestListener performanceTestListener) {
+        this.testRunDevice = testRunDevice;
+        this.testRunDeviceOrchestrator = testRunDeviceOrchestrator;
+        this.agentManagementService = agentManagementService;
+        this.testRun = testRun;
+        this.testTask = testTask;
+        this.logger = testRun.getLogger();
+        this.pkgName = testTask.getPkgName();
+        this.performanceTestListener = performanceTestListener;
+    }
+
+    public void testRunStarted() {
+        infoLogEnter("testRunStarted", "maestro test");
+
+        testRun.setTestStartTimeMillis(System.currentTimeMillis());
+        testRun.addNewTimeTag("testRunStarted", System.currentTimeMillis() - recordingStartTimeMillis);
+        testRunDeviceOrchestrator.setRunningTestName(testRunDevice, "MaestroTest.testRunStarted");
+        performanceTestListener.testRunStarted();
+        performanceTestListener.testStarted("MaestroTestCase" + index);
+        testRunDeviceOrchestrator.addGifFrameAsyncDelay(testRunDevice, agentManagementService.getScreenshotDir(), 5, logger);
+    }
+
+    private void initUnitCase(String caseName, int testSeconds) {
+        final int unitIndex = index;
+        ongoingTestUnit = new AndroidTestUnit();
+        ongoingTestUnit.setNumtests(index);
+        ongoingTestUnit.setStartTimeMillis(System.currentTimeMillis() - testSeconds * 1000);
+        ongoingTestUnit.setRelStartTimeInVideo(ongoingTestUnit.getStartTimeMillis() - testSeconds * 1000 - recordingStartTimeMillis);
+        ongoingTestUnit.setCurrentIndexNum(unitIndex);
+        ongoingTestUnit.setTestName(caseName);
+        ongoingTestUnit.setTestedClass("MaestroTest");
+        ongoingTestUnit.setDeviceTestResultId(testRun.getId());
+        ongoingTestUnit.setTestTaskId(testRun.getTestTaskId());
+
+        testRun.addNewTestUnit(ongoingTestUnit);
+        testRun.addNewTimeTag(unitIndex + ". " + ongoingTestUnit.getTitle(),
+                System.currentTimeMillis() - testSeconds * 1000 - recordingStartTimeMillis);
+        testRunDeviceOrchestrator.setRunningTestName(testRunDevice, ongoingTestUnit.getTitle());
+        testRunDeviceOrchestrator.addGifFrameAsyncDelay(testRunDevice, agentManagementService.getScreenshotDir(), 5, logger);
+        performanceTestListener.testStarted("MaestroTestCase" + index);
+    }
+
+    public void startRecording(int maxTime) {
+        logger.info("Start record screen");
+        if (!testTask.isDisableRecording()) {
+            testRunDeviceOrchestrator.startScreenRecorder(testRunDevice, testRun.getResultFolder(), maxTime <= 0 ? 30 * 60 : maxTime, logger);
+        }
+        logger.info("Start gif frames collection");
+        testRunDeviceOrchestrator.startGifEncoder(testRunDevice, testRun.getResultFolder(), pkgName + ".gif");
+        logger.info("Start logcat collection");
+        testRunDeviceOrchestrator.startLogCollector(testRunDevice, pkgName, testRun, logger);
+        testRun.setLogcatPath(agentManagementService.getTestBaseRelPathInUrl(new File(testRunDevice.getLogPath())));
+        recordingStartTimeMillis = System.currentTimeMillis();
+        final String initializing = "Initializing";
+        testRunDeviceOrchestrator.setRunningTestName(testRunDevice, initializing);
+        testRun.addNewTimeTag(initializing, 0);
+    }
+
+    public void testFailed(String caseName, int testSeconds, String error) {
+        errorLogEnter("testFailed", caseName);
+        performanceTestListener.testFailure("MaestroTestCase" + index);
+        index++;
+        initUnitCase(caseName, testSeconds);
+        ongoingTestUnit.setStack(error);
+        ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.FAILURE);
+        testRun.addNewTimeTag(ongoingTestUnit.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
+        testRun.addNewTimeTag(ongoingTestUnit.getTitle() + ".end", System.currentTimeMillis() - recordingStartTimeMillis);
+        testRun.oneMoreFailure();
+        ongoingTestUnit.setSuccess(false);
+        ongoingTestUnit.setEndTimeMillis(System.currentTimeMillis());
+        ongoingTestUnit.setRelEndTimeInVideo(ongoingTestUnit.getEndTimeMillis() - recordingStartTimeMillis);
+        numTests++;
+    }
+
+    public void testEnded(String caseName, int testSeconds) {
+        infoLogEnter("testEnded", caseName);
+        performanceTestListener.testSuccess("MaestroTestCase" + index);
+        index++;
+        initUnitCase(caseName, testSeconds);
+        testRun.addNewTimeTag(ongoingTestUnit.getTitle() + ".end",
+                System.currentTimeMillis() - recordingStartTimeMillis);
+        ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.OK);
+        ongoingTestUnit.setSuccess(true);
+        ongoingTestUnit.setEndTimeMillis(System.currentTimeMillis());
+        ongoingTestUnit.setRelEndTimeInVideo(ongoingTestUnit.getEndTimeMillis() - recordingStartTimeMillis);
+        numTests++;
+    }
+
+    public void testRunFailed(String outputPath) {
+        errorLogEnter("testRunFailed", "Start to copy output files", outputPath);
+        File file = new File(outputPath);
+        if (!file.exists()) {
+            logger.info("testRunFailed: " + outputPath + " not exist");
+            return;
+        }
+        File copiedFile = new File(testRun.getResultFolder(), file.getName());
+        copiedFile.mkdir();
+        FileUtil.copyFile(outputPath, copiedFile.getAbsolutePath());
+    }
+
+    public void testRunEnded() {
+        testRun.setTotalCount(numTests);
+        infoLogEnter("testRunEnded", Thread.currentThread().getName());
+        synchronized (this) {
+            if (alreadyEnd) {
+                return;
+            }
+            performanceTestListener.testRunFinished();
+            testRun.addNewTimeTag("testRunEnded", System.currentTimeMillis() - recordingStartTimeMillis);
+            testRun.onTestEnded();
+            testRunDeviceOrchestrator.setRunningTestName(testRunDevice, null);
+            testRunDeviceOrchestrator.stopGitEncoder(testRunDevice, agentManagementService.getScreenshotDir(), logger);
+            if (!testTask.isDisableRecording()) {
+                testRunDeviceOrchestrator.stopScreenRecorder(testRunDevice, testRun.getResultFolder(), logger);
+            }
+            testRunDeviceOrchestrator.stopLogCollector(testRunDevice);
+            alreadyEnd = true;
+        }
+    }
+
+    private void infoLogEnter(Object... args) {
+        StringBuilder builder = new StringBuilder();
+        for (Object arg : args) {
+            builder.append(" >").append(arg);
+        }
+        logger.info("TestRunListener: {}", builder);
+    }
+
+    private void errorLogEnter(Object... args) {
+        StringBuilder builder = new StringBuilder();
+        for (Object arg : args) {
+            builder.append(" >").append(arg);
+        }
+        logger.error("TestRunListener: {}", builder);
+    }
+
+    public File getGifFile() {
+        return testRunDevice.getGifFile();
+    }
+}

agent/src/main/java/com/microsoft/hydralab/agent/runner/maestro/MaestroResultReceiver.java

@@ -0,0 +1,96 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+package com.microsoft.hydralab.agent.runner.maestro;
+
+import com.microsoft.hydralab.common.util.Const;
+import com.microsoft.hydralab.common.util.LogUtils;
+import org.slf4j.Logger;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author zhoule
+ * @date 07/11/2023
+ */
+
+public class MaestroResultReceiver extends Thread {
+    private final InputStream inputStream;
+    private final Logger logger;
+    private final MaestroListener listener;
+    private static final String KEY_CASE_NAME = "caseName";
+    private static final String KEY_TEST_SECONDS = "testSeconds";
+    private static final String KEY_ERROR = "error";
+    private boolean isTestRunFailed = false;
+
+    public MaestroResultReceiver(InputStream inputStream, MaestroListener listener, Logger logger) {
+        this.inputStream = inputStream;
+        this.logger = logger;
+        this.listener = listener;
+    }
+
+    public void run() {
+        listener.testRunStarted();
+        try {
+            InputStreamReader isr = new InputStreamReader(inputStream, "UTF-8");
+            BufferedReader bufferedReader = new BufferedReader(isr);
+            String line;
+            while ((line = bufferedReader.readLine()) != null) {
+                logger.info(line);
+                if (line.startsWith("[Passed]")) {
+                    Map<String, String> caseInfo = analysisCaseInfo(line);
+                    listener.testEnded(caseInfo.get(KEY_CASE_NAME), Integer.parseInt(caseInfo.get(KEY_TEST_SECONDS)));
+                } else if (line.startsWith("[Failed]")) {
+                    Map<String, String> caseInfo = analysisCaseInfo(line);
+                    listener.testFailed(caseInfo.get(KEY_CASE_NAME), Integer.parseInt(caseInfo.get(KEY_TEST_SECONDS)), caseInfo.get(KEY_ERROR));
+                } else if (line.contains("Debug output")) {
+                    isTestRunFailed = true;
+                    logger.info("Start to analysis debug output");
+                }
+                if (isTestRunFailed) {
+                    if (LogUtils.isLegalStr(line, Const.RegexString.LINUX_ABSOLUTE_PATH, false)
+                            || LogUtils.isLegalStr(line, Const.RegexString.WINDOWS_ABSOLUTE_PATH, false)) {
+                        listener.testRunFailed(line);
+                    }
+                }
+            }
+            listener.testRunEnded();
+            isr.close();
+            bufferedReader.close();
+        } catch (IOException e) {
+            logger.info("Exception:" + e);
+            e.printStackTrace();
+        } finally {
+            synchronized (this) {
+                notify();
+            }
+        }
+    }
+
+    private Map<String, String> analysisCaseInfo(String line) {
+        Map<String, String> infoMap = new HashMap<>();
+        String[] msg = line.split(" ");
+        if (msg.length < 3) {
+            infoMap.put(KEY_CASE_NAME, "caseParseError");
+            infoMap.put(KEY_TEST_SECONDS, "0");
+        }
+        infoMap.put(KEY_CASE_NAME, msg[1]);
+        String testSeconds = msg[2].replace("s", "").replace("(", "").replace(")", "");
+        try {
+            Integer.parseInt(testSeconds);
+        } catch (NumberFormatException e) {
+            logger.info("Exception:" + e);
+            testSeconds = "0";
+        }
+        infoMap.put(KEY_TEST_SECONDS, testSeconds);
+        if (msg.length >= 4) {
+            infoMap.put(KEY_ERROR, line.substring(line.indexOf(msg[3])));
+        }
+        return infoMap;
+    }
+}