Skip to content

Commit

Permalink
TECH: Screenshot comparison tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Nikitae57 committed Jul 25, 2024
1 parent 40b6cdd commit 6065dd4
Show file tree
Hide file tree
Showing 34 changed files with 581 additions and 22 deletions.
1 change: 1 addition & 0 deletions allure-support/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ dependencies {
api(libs.bundles.allure)

implementation(projects.kaspresso)
implementation(projects.adbServer.adbServerCommon)

implementation(libs.kotlinStdlib)
implementation(libs.truth)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.kaspersky.components.alluresupport

import com.kaspersky.adbserver.common.log.logger.LogLevel
import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider
import com.kaspersky.components.alluresupport.files.resources.impl.AllureResourceFilesProvider
import com.kaspersky.components.alluresupport.files.resources.impl.DefaultAllureResourcesRootDirsProvider
Expand All @@ -10,17 +11,27 @@ import com.kaspersky.components.alluresupport.interceptors.testrun.DumpViewsTest
import com.kaspersky.components.alluresupport.interceptors.testrun.HackyVideoRecordingTestInterceptor
import com.kaspersky.components.alluresupport.interceptors.testrun.ScreenshotTestInterceptor
import com.kaspersky.components.alluresupport.interceptors.testrun.VideoRecordingTestInterceptor
import com.kaspersky.components.alluresupport.interceptors.testrun.VisualTestLateFailInterceptor
import com.kaspersky.components.alluresupport.results.AllureResultsHack
import com.kaspersky.components.alluresupport.runlisteners.AllureRunListener
import com.kaspersky.components.alluresupport.visual.AllureScreenshotsComparator
import com.kaspersky.components.alluresupport.visual.AllureVisualTestWatcher
import com.kaspersky.kaspresso.BuildConfig
import com.kaspersky.kaspresso.device.Device
import com.kaspersky.kaspresso.device.files.FilesImpl
import com.kaspersky.kaspresso.device.server.AdbServerImpl
import com.kaspersky.kaspresso.files.dirs.DefaultDirsProvider
import com.kaspersky.kaspresso.files.resources.impl.DefaultResourceFileNamesProvider
import com.kaspersky.kaspresso.files.resources.impl.DefaultResourceFilesProvider
import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirNameProvider
import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirsProvider
import com.kaspersky.kaspresso.instrumental.InstrumentalDependencyProvider
import com.kaspersky.kaspresso.kaspresso.Kaspresso
import com.kaspersky.kaspresso.logger.UiTestLoggerImpl
import com.kaspersky.kaspresso.runner.listener.addUniqueListener
import com.kaspersky.kaspresso.runner.listener.getUniqueListener
import com.kaspersky.kaspresso.visual.VisualTestParams
import com.kaspersky.kaspresso.visual.VisualTestType

/**
* Kaspresso Builder that includes all appropriate interceptors to support rich Allure reports.
Expand Down Expand Up @@ -64,6 +75,7 @@ fun Kaspresso.Builder.addAllureSupport(): Kaspresso.Builder = apply {
*/
fun Kaspresso.Builder.Companion.withForcedAllureSupport(
shouldRecordVideo: Boolean = true,
visualTestParams: VisualTestParams = VisualTestParams(testType = VisualTestType.valueOf(BuildConfig.VISUAL_TEST_TYPE)),
customize: Kaspresso.Builder.() -> Unit = {}
): Kaspresso.Builder = simple {
if (!isAndroidRuntime) {
Expand All @@ -72,6 +84,7 @@ fun Kaspresso.Builder.Companion.withForcedAllureSupport(
customize.invoke(this)
val instrumentalDependencyProvider = instrumentalDependencyProviderFactory.getComponentProvider<Kaspresso>(instrumentation)
forceAllureSupportFileProviders(instrumentalDependencyProvider)
initVisualTestParams(visualTestParams)
addRunListenersIfNeeded(instrumentalDependencyProvider)
}.apply {
postInitAllure(shouldRecordVideo, builder = this)
Expand All @@ -98,14 +111,26 @@ private fun Kaspresso.Builder.forceAllureSupportFileProviders(provider: Instrume
resourceFilesProvider = allureResourcesFilesProvider
}

private fun Kaspresso.Builder.initVisualTestParams(visualParams: VisualTestParams) {
visualTestParams = visualParams
testLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_TEST_LOGGER_TAG)
libLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_LIB_LOGGER_TAG)

screenshotsComparator = AllureScreenshotsComparator(visualTestParams, testLogger, dirsProvider, resourcesRootDirsProvider)
adbServer = AdbServerImpl(LogLevel.WARN, libLogger)
files = FilesImpl(libLogger, adbServer)
visualTestWatcher = AllureVisualTestWatcher(visualTestParams, testLogger, (dirsProvider as AllureDirsProvider), resourcesRootDirsProvider, files)
}

private fun Kaspresso.Builder.addRunListenersIfNeeded(provider: InstrumentalDependencyProvider) {
provider.runNotifier.apply {
addUniqueListener(::AllureRunListener)
addUniqueListener {
AllureResultsHack(
uiDevice = provider.uiDevice,
resourcesRootDirsProvider = resourcesRootDirsProvider as DefaultAllureResourcesRootDirsProvider,
dirsProvider = dirsProvider as AllureDirsProvider
dirsProvider = dirsProvider as AllureDirsProvider,
visualTestParams = visualTestParams,
)
}
}
Expand All @@ -126,6 +151,7 @@ private fun postInitAllure(shouldRecordVideo: Boolean, builder: Kaspresso.Builde
DumpLogcatTestInterceptor(logcatDumper),
ScreenshotTestInterceptor(screenshots),
DumpViewsTestInterceptor(viewHierarchyDumper),
VisualTestLateFailInterceptor(),
)
)
if (shouldRecordVideo) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package com.kaspersky.components.alluresupport.interceptors.testrun

import com.kaspersky.components.alluresupport.results.AllureVisualTestFlag
import com.kaspersky.kaspresso.device.screenshots.ScreenshotsImpl
import com.kaspersky.kaspresso.interceptors.watcher.testcase.TestRunWatcherInterceptor
import com.kaspersky.kaspresso.testcases.models.info.TestInfo

class VisualTestLateFailInterceptor : TestRunWatcherInterceptor {
override fun onAfterSectionStarted(testInfo: TestInfo) {
if (AllureVisualTestFlag.shouldFailLate.get()) {
throw ScreenshotsImpl.ScreenshotDoesntMatchException("There were failed screenshot comparisons. Check the allure report")
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,17 @@ import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider
import com.kaspersky.components.alluresupport.files.resources.AllureResourcesRootDirsProvider
import com.kaspersky.components.alluresupport.files.resources.impl.AllureResourceFilesProvider
import com.kaspersky.kaspresso.runner.listener.KaspressoRunListener
import com.kaspersky.kaspresso.visual.VisualTestParams
import com.kaspersky.kaspresso.visual.VisualTestType
import io.qameta.allure.kotlin.Allure
import org.junit.runner.Result
import java.io.File

class AllureResultsHack(
private val uiDevice: UiDevice,
private val visualTestParams: VisualTestParams,
resourcesRootDirsProvider: AllureResourcesRootDirsProvider,
dirsProvider: AllureDirsProvider,
private val dirsProvider: AllureDirsProvider,
) : KaspressoRunListener {

private val allureResultsSourceDir: File =
Expand Down Expand Up @@ -48,6 +51,13 @@ class AllureResultsHack(

allureResultsSourceDir.deleteRecursively()
stubVideosDir.deleteRecursively()

if (visualTestParams.testType == VisualTestType.Record) {
val rootDir = dirsProvider.provideNew(File("")).absolutePath
val newScreenshotsDir = File(rootDir, File(visualTestParams.hostScreenshotsDir).name)
val targetScreenshotsDir = dirsProvider.provideNewOnSdCard(File(visualTestParams.hostScreenshotsDir))
newScreenshotsDir.copyRecursively(targetScreenshotsDir)
}
}

data class VideoBinding(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package com.kaspersky.components.alluresupport.results

import java.util.concurrent.atomic.AtomicBoolean

// TODO(Nikita Evdokimov) - Certainly there should be a better way
/**
* @see com.kaspersky.components.alluresupport.interceptors.testrun.VisualTestLateFailInterceptor
*/
object AllureVisualTestFlag {
val shouldFailLate = AtomicBoolean(false)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package com.kaspersky.components.alluresupport.visual

import android.graphics.Bitmap
import com.kaspersky.components.alluresupport.files.attachScreenshotToAllureReport
import com.kaspersky.kaspresso.files.dirs.DirsProvider
import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider
import com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator
import com.kaspersky.kaspresso.logger.Logger
import com.kaspersky.kaspresso.visual.VisualTestParams
import java.io.File

class AllureScreenshotsComparator(
visualTestParams: VisualTestParams,
logger: Logger,
dirsProvider: DirsProvider,
resourcesRootDirsProvider: ResourcesRootDirsProvider,
) : DefaultScreenshotsComparator(visualTestParams, logger, dirsProvider, resourcesRootDirsProvider) {
override fun compare(originalScreenshot: File, newScreenshot: File): Boolean {
val doScreenshotsMatch = super.compare(originalScreenshot, newScreenshot)
if (!doScreenshotsMatch) {
originalScreenshot.attachScreenshotToAllureReport()
newScreenshot.attachScreenshotToAllureReport()
}

return doScreenshotsMatch
}

override fun processScreenshotDiff(original: Bitmap, diffPixels: IntArray, diffName: String): File {
return super.processScreenshotDiff(original, diffPixels, diffName).also {
it.attachScreenshotToAllureReport()
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package com.kaspersky.components.alluresupport.visual

import com.kaspersky.components.alluresupport.results.AllureVisualTestFlag
import com.kaspersky.components.alluresupport.withForcedAllureSupport
import com.kaspersky.kaspresso.device.screenshots.ScreenshotsImpl
import com.kaspersky.kaspresso.kaspresso.Kaspresso
import com.kaspersky.kaspresso.testcases.api.testcase.VisualTestCase
import io.qameta.allure.kotlin.Allure
import io.qameta.allure.kotlin.model.Status
import io.qameta.allure.kotlin.model.StatusDetails

abstract class AllureVisualTestCase(
private val failEarly: Boolean = false,
kaspressoBuilder: Kaspresso.Builder = Kaspresso.Builder.withForcedAllureSupport()
) : VisualTestCase(kaspressoBuilder = kaspressoBuilder) {

override fun assertScreenshot(tag: String, isFullWindow: Boolean) {
try {
device.screenshots.assert(tag, isFullWindow)
} catch (ex: ScreenshotsImpl.ScreenshotDoesntMatchException) {
if (failEarly) { throw ex }

Allure.lifecycle.updateStep {
it.status = Status.FAILED
it.statusDetails = StatusDetails(known = true, muted = true, message = ex.message, trace = ex.stackTraceToString())
}
Allure.lifecycle.stopStep()
AllureVisualTestFlag.shouldFailLate.set(true)
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package com.kaspersky.components.alluresupport.visual

import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider
import com.kaspersky.kaspresso.device.files.Files
import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider
import com.kaspersky.kaspresso.logger.Logger
import com.kaspersky.kaspresso.visual.VisualTestParams
import com.kaspersky.kaspresso.visual.VisualTestType
import com.kaspersky.kaspresso.visual.VisualTestWatcher
import java.io.File

class AllureVisualTestWatcher(
private val params: VisualTestParams,
private val logger: Logger,
private val dirsProvider: AllureDirsProvider,
resourcesRootDirsProvider: ResourcesRootDirsProvider,
private val files: Files,
) : VisualTestWatcher {

private val diffDir = dirsProvider.provideNew(resourcesRootDirsProvider.screenshotsDiffRootDir)
private val originalScreenshotsTargetDir: File
get() {
val rootDir = dirsProvider.provideNewOnSdCard(File("")).absolutePath
return File(rootDir, File(params.hostScreenshotsDir).name)
}

override fun prepare() {
logger.i("Visual test run started. Parameters: $params")

if (params.testType == VisualTestType.Compare) {
logger.i("Pushing the screenshots unto the device...")
dirsProvider.provideCleared(diffDir)

// Allure stores all files in the app's private directory. We can't "adb push" directly there,
// so we have to do this in 2 steps
dirsProvider.provideCleared(originalScreenshotsTargetDir)
val tmp = dirsProvider.provideNewOnSdCard(File(""))
files.push(params.hostScreenshotsDir, tmp.absolutePath)
val target = dirsProvider.provideNew(File("")).resolve(params.hostScreenshotsDir)
File(tmp, params.hostScreenshotsDir).copyRecursively(target, overwrite = true)
logger.i("Done pushing the screenshots unto the device")
}
}

override fun cleanUp() {
// Do nothing
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,6 @@ configure<BaseExtension> {
resValues = false
shaders = false
viewBinding = false
buildConfig = true
}
}
1 change: 1 addition & 0 deletions gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ org.gradle.caching=true

kotlin.code.style=official

android.injected.androidTest.leaveApksInstalledAfterRun=true
android.useAndroidX = true

kaspresso.version=1.5.4
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ thirdPartyReport = "0.19.1035"
[libraries]
# plugins
kotlinPlugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" }
androidPlugin = "com.android.tools.build:gradle:8.1.4"
androidPlugin = "com.android.tools.build:gradle:8.2.0"
versionsPlugin = "com.github.ben-manes:gradle-versions-plugin:0.50.0"
thirdPartyReportPlugin = { module = "com.kaspersky.gradle:third-party-report", version.ref = "thirdPartyReport" }
airPlugin = { module = "com.kaspersky.gradle:air", version.ref = "thirdPartyReport" }
Expand Down
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
4 changes: 4 additions & 0 deletions kaspresso/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ plugins {

android {
namespace = "com.kaspersky.kaspresso"

defaultConfig {
buildConfigField("String", "VISUAL_TEST_TYPE", System.getenv("VISUAL_TEST_TYPE") ?: property("kaspresso.visualTestType").toString()) // [Record, Compare]
}
}

publish {
Expand Down
3 changes: 2 additions & 1 deletion kaspresso/gradle.properties
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
publish.artifactGroup=com.kaspersky.android-components
publish.artifactName=kaspresso
publish.publicationName=Kaspresso
publish.bintrayRepo=Kaspresso
publish.bintrayRepo=Kaspresso
kaspresso.visualTestType="Compare"
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import com.kaspersky.kaspresso.logger.UiTestLogger
*/
class FilesImpl(
private val logger: UiTestLogger,
private val adbServer: AdbServer
private val adbServer: AdbServer,
) : Files {

/**
Expand All @@ -20,7 +20,7 @@ class FilesImpl(
* @param devicePath a path to copy.
*/
override fun push(serverPath: String, devicePath: String) {
adbServer.performAdb("push $serverPath $devicePath")
adbServer.performAdb("push", listOf(serverPath, devicePath))
logger.i("Push file from $serverPath to $devicePath")
}

Expand All @@ -32,7 +32,7 @@ class FilesImpl(
* @param path a path to remove
*/
override fun remove(path: String) {
adbServer.performShell("rm -f $path")
adbServer.performShell("rm", listOf("-rf", path))
logger.i("Remove file from $path")
}

Expand All @@ -45,8 +45,8 @@ class FilesImpl(
* @param serverPath a path to copy. (If empty - pulls in adbServer directory (folder with file "adbserver-desktop.jar"))
*/
override fun pull(devicePath: String, serverPath: String) {
adbServer.performCmd("mkdir -p $serverPath")
adbServer.performAdb("pull $devicePath $serverPath")
adbServer.performCmd("mkdir", listOf("-p", serverPath))
adbServer.performAdb("pull", listOf(devicePath, serverPath))
logger.i("Pull file from $devicePath to $serverPath")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,8 @@ interface Screenshots {
* @param tag a unique tag to further identify the screenshot. Must match [a-zA-Z0-9_-]+.
*/
fun takeFullWindowAndApply(tag: String, block: File.() -> Unit)

fun assert(tag: String, isFullWindow: Boolean)

fun assertAndApply(tag: String, isFullWindow: Boolean, block: File.() -> Unit)
}
Loading

0 comments on commit 6065dd4

Please sign in to comment.