Run tests in Cirrus CI again

.cirrus.yml

@@ -1,13 +1,66 @@
-task:
-  name: Build with AOSP
-  only_if: $CIRRUS_PR_LABELS =~ ".*aosp-build.*"
-  timeout_in: 70m
-  container:
-    image: ubuntu:23.04
-    cpu: 8
-    memory: 32G
-  build_script:
-    - ./.github/scripts/build_aosp.sh aosp_arm64 ap1a userdebug android-14.0.0_r29
+container:
+  image: ghcr.io/cirruslabs/android-sdk:34
+  kvm: true
+  cpu: 8
+  memory: 16G
+
+instrumentation_tests_task:
+  name: "Cirrus CI Instrumentation Tests"
+  start_avd_background_script:
+    sdkmanager --install "system-images;android-34;default;x86_64" "emulator";
+    echo no | avdmanager create avd -n seedvault -k "system-images;android-34;default;x86_64";
+    $ANDROID_HOME/emulator/emulator
+    -avd seedvault
+    -no-audio
+    -no-boot-anim
+    -gpu swiftshader_indirect
+    -no-snapshot
+    -no-window
+    -writable-system;
+  provision_avd_background_script:
+    wget https://github.com/seedvault-app/seedvault-test-data/releases/download/3/backup.tar.gz;
+
+    adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
+    adb root;
+    sleep 5;
+    adb remount;
+    adb reboot;
+    adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
+    adb root;
+    sleep 5;
+    adb remount;
+    sleep 5;
+  assemble_script:
+    ./gradlew :app:assembleRelease :contacts:assembleRelease assembleAndroidTest
+  install_app_script:
+    timeout 180s bash -c 'while [[ -z $(adb shell mount | grep "/system " | grep "(rw,") ]]; do sleep 1; done;';
+    adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
+
+    adb shell mkdir -p /sdcard/seedvault_baseline;
+    adb push backup.tar.gz /sdcard/seedvault_baseline/backup.tar.gz;
+    adb shell tar xzf /sdcard/seedvault_baseline/backup.tar.gz --directory=/sdcard/seedvault_baseline;
+
+    adb shell mkdir -p /system/priv-app/Seedvault;
+    adb push app/build/outputs/apk/release/app-release.apk /system/priv-app/Seedvault/Seedvault.apk;
+    adb push permissions_com.stevesoltys.seedvault.xml /system/etc/permissions/privapp-permissions-seedvault.xml;
+    adb push allowlist_com.stevesoltys.seedvault.xml /system/etc/sysconfig/allowlist-seedvault.xml;
+
+    adb shell mkdir -p /system/priv-app/ContactsBackup;
+    adb push contactsbackup/build/outputs/apk/release/contactsbackup-release.apk /system/priv-app/ContactsBackup/contactsbackup.apk;
+    adb push contactsbackup/default-permissions_org.calyxos.backup.contacts.xml /system/etc/default-permissions/default-permissions_org.calyxos.backup.contacts.xml;
+
+    adb shell bmgr enable true;
+    adb reboot;
+    adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
+    adb shell bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport;
+    adb reboot;
+    adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done;';
+  run_large_tests_script: ./gradlew -Pandroid.testInstrumentationRunnerArguments.size=large :app:connectedAndroidTest
+  run_other_tests_script: ./gradlew -Pandroid.testInstrumentationRunnerArguments.notAnnotation=androidx.test.filters.LargeTest connectedAndroidTest
   always:
-    seedvault_artifacts:
-      path: Seedvault.apk
+    pull_screenshots_script:
+      adb pull /sdcard/seedvault_test_results
+    screenshots_artifacts:
+      path: "seedvault_test_results/**/*.mp4"
+    logcat_artifacts:
+      path: "seedvault_test_results/**/*.log"

app/build.gradle.kts

@@ -32,13 +32,6 @@ android {
         versionNameSuffix = "-${gitDescribe()}"
         testInstrumentationRunner = "com.stevesoltys.seedvault.KoinInstrumentationTestRunner"
         testInstrumentationRunnerArguments["disableAnalytics"] = "true"
-
-        if (project.hasProperty("instrumented_test_size")) {
-            val testSize = project.property("instrumented_test_size").toString()
-            println("Instrumented test size: $testSize")
-
-            testInstrumentationRunnerArguments["size"] = testSize
-        }
     }
 
     signingConfigs {

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeBackupTestBase.kt

@@ -9,7 +9,6 @@ import android.content.pm.PackageInfo
 import android.os.ParcelFileDescriptor
 import androidx.test.uiautomator.Until
 import com.stevesoltys.seedvault.e2e.io.BackupDataInputIntercept
-import com.stevesoltys.seedvault.e2e.io.InputStreamIntercept
 import com.stevesoltys.seedvault.e2e.screen.impl.BackupScreen
 import com.stevesoltys.seedvault.transport.backup.FullBackup
 import com.stevesoltys.seedvault.transport.backup.InputFactory
@@ -21,8 +20,10 @@ import io.mockk.every
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withTimeout
+import org.calyxos.seedvault.core.toHexString
 import org.koin.core.component.get
-import java.io.ByteArrayOutputStream
+import java.security.DigestInputStream
+import java.security.MessageDigest
 import java.util.concurrent.atomic.AtomicBoolean
 import kotlin.test.fail
 
@@ -154,7 +155,8 @@ internal interface LargeBackupTestBase : LargeTestBase {
 
     private fun spyOnFullBackupData(backupResult: SeedvaultLargeTestResult) {
         var packageName: String? = null
-        var dataIntercept = ByteArrayOutputStream()
+        val messageDigest = MessageDigest.getInstance("SHA-256")
+        var digestInputStream: DigestInputStream? = null
 
         coEvery {
             spyFullBackup.performFullBackup(any(), any(), any())
@@ -166,20 +168,19 @@ internal interface LargeBackupTestBase : LargeTestBase {
         every {
             spyInputFactory.getInputStream(any())
         } answers {
-            InputStreamIntercept(
-                inputStream = callOriginal(),
-                intercept = dataIntercept
-            )
+            digestInputStream = DigestInputStream(callOriginal(), messageDigest)
+            digestInputStream!!
         }
 
         coEvery {
             spyFullBackup.finishBackup()
         } answers {
             val result = callOriginal()
-            backupResult.full[packageName!!] = dataIntercept.toByteArray().sha256()
+            val digest = digestInputStream?.messageDigest ?: fail("No digestInputStream")
+            backupResult.full[packageName!!] = digest.digest().toHexString()
 
             packageName = null
-            dataIntercept = ByteArrayOutputStream()
+            digest.reset()
             result
         }
     }
@@ -192,9 +193,6 @@ internal interface LargeBackupTestBase : LargeTestBase {
         every {
             spyBackupNotificationManager.onBackupSuccess(any(), any(), any())
         } answers {
-            val success = firstArg<Boolean>()
-            assert(success) { "Backup failed." }
-
             callOriginal()
             completed.set(true)
         }

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeRestoreTestBase.kt

@@ -8,12 +8,13 @@ package com.stevesoltys.seedvault.e2e
 import android.content.pm.PackageInfo
 import android.os.ParcelFileDescriptor
 import com.stevesoltys.seedvault.e2e.io.BackupDataOutputIntercept
-import com.stevesoltys.seedvault.e2e.io.OutputStreamIntercept
 import com.stevesoltys.seedvault.e2e.screen.impl.RecoveryCodeScreen
 import com.stevesoltys.seedvault.e2e.screen.impl.RestoreScreen
 import com.stevesoltys.seedvault.transport.restore.FullRestore
 import com.stevesoltys.seedvault.transport.restore.KVRestore
 import com.stevesoltys.seedvault.transport.restore.OutputFactory
+import io.mockk.Call
+import io.mockk.MockKAnswerScope
 import io.mockk.clearMocks
 import io.mockk.coEvery
 import io.mockk.every
@@ -22,8 +23,11 @@ import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import kotlinx.coroutines.withTimeout
+import org.calyxos.seedvault.core.toHexString
 import org.koin.core.component.get
-import java.io.ByteArrayOutputStream
+import java.security.DigestOutputStream
+import java.security.MessageDigest
+import kotlin.test.fail
 
 internal interface LargeRestoreTestBase : LargeTestBase {
 
@@ -161,14 +165,26 @@ internal interface LargeRestoreTestBase : LargeTestBase {
 
         clearMocks(spyKVRestore)
 
-        coEvery {
-            spyKVRestore.initializeState(any(), any(), any(), any())
-        } answers {
-            packageName = arg<PackageInfo>(3).packageName
+        fun initializeStateBlock(
+            packageInfoIndex: Int
+        ): MockKAnswerScope<Unit, Unit>.(Call) -> Unit = {
+            packageName = arg<PackageInfo>(packageInfoIndex).packageName
             restoreResult.kv[packageName!!] = mutableMapOf()
             callOriginal()
         }
 
+        coEvery {
+            spyKVRestore.initializeState(any(), any(), any(), any())
+        } answers initializeStateBlock(1)
+
+        coEvery {
+            spyKVRestore.initializeStateV1(any(), any(), any(), any())
+        } answers initializeStateBlock(2)
+
+        coEvery {
+            spyKVRestore.initializeStateV0(any(), any())
+        } answers initializeStateBlock(1)
+
         every {
             spyOutputFactory.getBackupDataOutput(any())
         } answers {
@@ -182,47 +198,61 @@ internal interface LargeRestoreTestBase : LargeTestBase {
 
     private fun spyOnFullRestoreData(restoreResult: SeedvaultLargeTestResult) {
         var packageName: String? = null
-        var dataIntercept = ByteArrayOutputStream()
+        val messageDigest = MessageDigest.getInstance("SHA-256")
+        var digestOutputStream: DigestOutputStream? = null
 
         clearMocks(spyFullRestore)
 
-        coEvery {
-            spyFullRestore.initializeState(any(), any(), any())
-        } answers {
+        fun initializeStateBlock(
+            packageInfoIndex: Int
+        ): MockKAnswerScope<Unit, Unit>.(Call) -> Unit = {
             packageName?.let {
-                restoreResult.full[it] = dataIntercept.toByteArray().sha256()
+                // sometimes finishRestore() doesn't get called, so get data from last package here
+                digestOutputStream?.messageDigest?.let { digest ->
+                    restoreResult.full[packageName!!] = digest.digest().toHexString()
+                }
             }
 
-            packageName = arg<PackageInfo>(3).packageName
-            dataIntercept = ByteArrayOutputStream()
+            packageName = arg<PackageInfo>(packageInfoIndex).packageName
 
             callOriginal()
         }
 
+        coEvery {
+            spyFullRestore.initializeState(any(), any(), any())
+        } answers initializeStateBlock(1)
+
+        coEvery {
+            spyFullRestore.initializeStateV1(any(), any(), any())
+        } answers initializeStateBlock(2)
+
+        coEvery {
+            spyFullRestore.initializeStateV0(any(), any())
+        } answers initializeStateBlock(1)
+
         every {
             spyOutputFactory.getOutputStream(any())
         } answers {
-            OutputStreamIntercept(
-                outputStream = callOriginal(),
-                intercept = dataIntercept
-            )
+            digestOutputStream = DigestOutputStream(callOriginal(), messageDigest)
+            digestOutputStream!!
         }
 
         every {
             spyFullRestore.abortFullRestore()
         } answers {
             packageName = null
-            dataIntercept = ByteArrayOutputStream()
+            digestOutputStream?.messageDigest?.reset()
             callOriginal()
         }
 
         every {
             spyFullRestore.finishRestore()
         } answers {
-            restoreResult.full[packageName!!] = dataIntercept.toByteArray().sha256()
+            val digest = digestOutputStream?.messageDigest ?: fail("No digestOutputStream")
+            restoreResult.full[packageName!!] = digest.digest().toHexString()
 
             packageName = null
-            dataIntercept = ByteArrayOutputStream()
+            digest.reset()
             callOriginal()
         }
     }

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/LargeTestBase.kt

@@ -49,14 +49,14 @@ internal interface LargeTestBase : KoinComponent {
 
     companion object {
         private const val TEST_STORAGE_FOLDER = "seedvault_test"
-        private const val TEST_VIDEO_FOLDER = "seedvault_test_results"
+        private const val TEST_RESULT_FOLDER = "seedvault_test_results"
     }
 
     val externalStorageDir: String get() = Environment.getExternalStorageDirectory().absolutePath
 
     val testStoragePath get() = "$externalStorageDir/$TEST_STORAGE_FOLDER"
 
-    val testVideoPath get() = "$externalStorageDir/$TEST_VIDEO_FOLDER"
+    val testResultPath get() = "$externalStorageDir/$TEST_RESULT_FOLDER"
 
     val targetContext: Context
         get() = InstrumentationRegistry.getInstrumentation().targetContext
@@ -123,7 +123,7 @@ internal interface LargeTestBase : KoinComponent {
         keepRecordingScreen: AtomicBoolean,
         testName: String,
     ) {
-        val folder = testVideoPath
+        val folder = testResultPath
         runCommand("mkdir -p $folder")
 
         val fileName = testResultFilename(testName)
@@ -149,7 +149,7 @@ internal interface LargeTestBase : KoinComponent {
 
         // write logcat to file
         val fileName = testResultFilename(testName)
-        runCommand("logcat -d -f $testVideoPath/$fileName.log")
+        runCommand("logcat -d -f $testResultPath/$fileName.log")
     }
 
     fun uninstallPackages(packages: Collection<PackageInfo>) {
@@ -162,7 +162,7 @@ internal interface LargeTestBase : KoinComponent {
 
     fun clearTestBackups() {
         File(testStoragePath).deleteRecursively()
-        File(testVideoPath).deleteRecursively()
+        File(testResultPath).deleteRecursively()
     }
 
     fun changeBackupLocation(
@@ -225,6 +225,7 @@ internal interface LargeTestBase : KoinComponent {
 
     fun confirmCode() {
         RecoveryCodeScreen {
+            startNewBackupButton.click()
             confirmCodeButton.click()
 
             verifyCodeButton.scrollTo().click()

app/src/androidTest/java/com/stevesoltys/seedvault/e2e/SeedvaultLargeTest.kt

@@ -45,9 +45,8 @@ internal abstract class SeedvaultLargeTest :
         clearTestBackups()
 
         runCommand("bmgr enable true")
-        sleep(60_000)
         runCommand("bmgr transport com.stevesoltys.seedvault.transport.ConfigurableBackupTransport")
-        sleep(60_000)
+        sleep(5000)
 
         startRecordingTest(keepRecordingScreen, name.methodName)
         restoreBaselineBackup()