diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh new file mode 100755 index 000000000..e091cbad1 --- /dev/null +++ b/.github/scripts/run_tests.sh @@ -0,0 +1,38 @@ +adb reboot +adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;' + +adb root +sleep 5 +adb remount + +echo "Installing Seedvault app..." +adb shell mkdir -p /system/priv-app/Seedvault +adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk + +echo "Installing Seedvault permissions..." +adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml +adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml + +echo "Setting Seedvault transport..." +sleep 10 +adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport + +large_test_exit_code=0 +./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$? + +adb pull /sdcard/seedvault_test_results + +if [ "$large_test_exit_code" -ne 0 ]; then + echo 'Large tests failed.' + exit 1 +fi + +medium_test_exit_code=0 +./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$? + +if [ "$medium_test_exit_code" -ne 0 ]; then + echo 'Medium tests failed.' + exit 1 +fi + +exit 0 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82ca5c423..52dd750b7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,10 @@ name: Build on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: build: name: Build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 925a65b10..330834d2b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,14 +13,20 @@ concurrency: jobs: instrumentation_tests: runs-on: macos-11 + if: github.repository == 'seedvault-app/seedvault' + timeout-minutes: 80 strategy: fail-fast: false matrix: - android_target: [33, 34] - emulator_type: [default, google_apis] + android_target: [ 33, 34 ] + emulator_type: [ default, google_apis, google_atd ] exclude: - android_target: 34 emulator_type: default + - android_target: 34 + emulator_type: google_atd + - android_target: 33 + emulator_type: google_apis steps: - name: Checkout Code uses: actions/checkout@v3 @@ -31,32 +37,9 @@ jobs: java-version: '17' cache: 'gradle' - - name: AVD cache - uses: actions/cache@v3 - id: avd-cache - with: - path: | - ~/.android/avd/* - ~/.android/adb* - key: aosp-${{ matrix.emulator_type }}-${{ matrix.android_target }}-${{ runner.os }} - - name: Build Release APK run: ./gradlew :app:assembleRelease - - name: Create AVD snapshot - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: ${{ matrix.android_target }} - target: ${{ matrix.emulator_type }} - arch: x86_64 - force-avd-creation: false - emulator-options: -writable-system -no-snapshot-load -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - disable-animations: true - script: | - ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64" - echo "Generated AVD snapshot for caching." - - name: Assemble tests run: ./gradlew :app:assembleAndroidTest @@ -66,53 +49,18 @@ jobs: api-level: ${{ matrix.android_target }} target: ${{ matrix.emulator_type }} arch: x86_64 - force-avd-creation: false - emulator-options: -writable-system -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none - profile: pixel_6a - heap-size: '512M' - ram-size: '4096M' + force-avd-creation: true + emulator-options: -cores 2 -writable-system -no-snapshot-load -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disk-size: '14G' sdcard-path-or-size: '4096M' - cores: 3 - disable-animations: false + disable-animations: true script: | - adb root - sleep 5 - adb remount - - echo "Installing Seedvault app..." - adb shell mkdir -p /system/priv-app/Seedvault - adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk - - echo "Installing Seedvault permissions..." - adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml - adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml - - echo "Setting Seedvault transport..." - sleep 10 - adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport - - wget --quiet https://github.com/seedvault-app/seedvault-test-data/releases/download/1/backup.tar.gz - adb shell mkdir -p /sdcard/seedvault_baseline - adb push backup.tar.gz /sdcard/seedvault_baseline - adb wait-for-device - adb shell tar xzf /sdcard/seedvault_baseline/backup.tar.gz --directory=/sdcard/seedvault_baseline - adb shell rm /sdcard/seedvault_baseline/backup.tar.gz - - large_test_exit_code=0 - ./gradlew --stacktrace -Pinstrumented_test_size=large :app:connectedAndroidTest || large_test_exit_code=$? - - medium_test_exit_code=0 - ./gradlew --stacktrace -Pinstrumented_test_size=medium :app:connectedAndroidTest || medium_test_exit_code=$? - - adb pull /sdcard/seedvault_test_videos - - if [ $large_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi - if [ $medium_test_exit_code -ne 0 ]; then echo 'Gradle test failed.'; exit 0; fi + ./app/development/scripts/provision_emulator.sh "test" "system-images;android-${{ matrix.android_target }};${{ matrix.emulator_type }};x86_64" + ./.github/scripts/run_tests.sh - - name: Upload screenshots and videos + - name: Upload test results if: always() uses: actions/upload-artifact@v3 with: - name: seedvault_test_videos - path: seedvault_test_videos/**/*.mp4 + name: ${{ matrix.emulator_type }}-${{ matrix.android_target }}-results + path: seedvault_test_results/**/* diff --git a/app/build.gradle b/app/build.gradle index 56508f332..6488da32e 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -162,7 +162,7 @@ dependencies { androidTestImplementation 'androidx.test:runner:1.4.0' androidTestImplementation 'androidx.test:rules:1.4.0' androidTestImplementation 'androidx.test.ext:junit:1.1.3' - androidTestImplementation "io.mockk:mockk-android:$mockk_version" + androidTestImplementation "io.mockk:mockk-android:1.13.8" androidTestImplementation 'androidx.test.uiautomator:uiautomator:2.2.0' } diff --git a/app/development/scripts/provision_emulator.sh b/app/development/scripts/provision_emulator.sh index cfdcc8871..fef04b8bb 100755 --- a/app/development/scripts/provision_emulator.sh +++ b/app/development/scripts/provision_emulator.sh @@ -20,7 +20,7 @@ DEVELOPMENT_DIR=$SCRIPT_DIR/.. ROOT_PROJECT_DIR=$SCRIPT_DIR/../../.. echo "Downloading system image..." -$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE" +yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "$SYSTEM_IMAGE" # create AVD if it doesn't exist if $ANDROID_HOME/cmdline-tools/latest/bin/avdmanager list avd | grep -q "$EMULATOR_NAME"; then diff --git a/app/development/scripts/start_emulator.sh b/app/development/scripts/start_emulator.sh index 4c071184d..b2113487f 100755 --- a/app/development/scripts/start_emulator.sh +++ b/app/development/scripts/start_emulator.sh @@ -14,9 +14,5 @@ fi EMULATOR_NAME=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -DEVELOPMENT_DIR=$SCRIPT_DIR/.. -ROOT_PROJECT_DIR=$SCRIPT_DIR/../../.. - echo "Starting emulator..." nohup $ANDROID_HOME/emulator/emulator -avd "$EMULATOR_NAME" -gpu swiftshader_indirect -writable-system >/dev/null 2>&1 & diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt index b386214fa..86f14a28a 100644 --- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt @@ -44,7 +44,7 @@ internal interface LargeTestBase : KoinComponent { companion object { private const val TEST_STORAGE_FOLDER = "seedvault_test" - private const val TEST_VIDEO_FOLDER = "seedvault_test_videos" + private const val TEST_VIDEO_FOLDER = "seedvault_test_results" } val externalStorageDir: String get() = Environment.getExternalStorageDirectory().absolutePath @@ -106,19 +106,23 @@ internal interface LargeTestBase : KoinComponent { uiAutomation.executeShellCommand(command).close() } + fun testResultFilename(testName: String): String { + val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss") + val timeStamp = simpleDateFormat.format(Calendar.getInstance().time) + return "${timeStamp}_${testName.replace(" ", "_")}" + } + @OptIn(DelicateCoroutinesApi::class) @WorkerThread - suspend fun startScreenRecord( + suspend fun startRecordingTest( keepRecordingScreen: AtomicBoolean, testName: String, ) { - val simpleDateFormat = SimpleDateFormat("yyyyMMdd_hhmmss") - val timeStamp = simpleDateFormat.format(Calendar.getInstance().time) - val fileName = "${timeStamp}_${testName.replace(" ", "_")}" - val folder = testVideoPath runCommand("mkdir -p $folder") + val fileName = testResultFilename(testName) + // screen record automatically stops after 3 minutes // we need to block on a loop and split it into multiple files GlobalScope.launch(Dispatchers.IO) { @@ -131,10 +135,16 @@ internal interface LargeTestBase : KoinComponent { } @WorkerThread - fun stopScreenRecord(keepRecordingScreen: AtomicBoolean) { + fun stopRecordingTest( + keepRecordingScreen: AtomicBoolean, + testName: String, + ) { keepRecordingScreen.set(false) - runCommand("pkill -2 screenrecord") + + // write logcat to file + val fileName = testResultFilename(testName) + runCommand("logcat -d -f $testVideoPath/$fileName.log") } fun uninstallPackages(packages: Collection) { diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt index 423c461ec..2d2be5f19 100644 --- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt @@ -38,13 +38,13 @@ internal abstract class SeedvaultLargeTest : resetApplicationState() clearTestBackups() - startScreenRecord(keepRecordingScreen, name.methodName) + startRecordingTest(keepRecordingScreen, name.methodName) restoreBaselineBackup() } @After open fun tearDown() { - stopScreenRecord(keepRecordingScreen) + stopRecordingTest(keepRecordingScreen, name.methodName) } /** diff --git a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt index fdcdd67b9..bc13414fa 100644 --- a/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt +++ b/app/src/androidTest/java/com/stevesoltys/seedvault/e2e/screen/UiDeviceScreen.kt @@ -11,6 +11,10 @@ import java.lang.Thread.sleep abstract class UiDeviceScreen { + companion object { + private const val SELECTOR_TIMEOUT = 180000L + } + operator fun invoke(function: T.() -> Unit) { function.invoke(this as T) } @@ -18,8 +22,11 @@ abstract class UiDeviceScreen { fun UiObject.scrollTo( scrollSelector: UiSelector = UiSelector().className(ScrollView::class.java), ): UiObject { - UiScrollable(scrollSelector).scrollIntoView(this) - waitForExists(15000) + val uiScrollable = UiScrollable(scrollSelector) + uiScrollable.waitForExists(SELECTOR_TIMEOUT) + uiScrollable.scrollIntoView(this) + waitForExists(SELECTOR_TIMEOUT) + sleep(2000) return this } @@ -32,6 +39,6 @@ abstract class UiDeviceScreen { private fun device() = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()) .also { - Configurator.getInstance().waitForSelectorTimeout = 60000 + Configurator.getInstance().waitForSelectorTimeout = SELECTOR_TIMEOUT } }