Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add setup for functional tests. Test sending events over the local variable limit. #85

Merged
merged 14 commits into from
Oct 30, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .github/workflows/readme.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,33 @@ jobs:
with:
name: artifact
path: ~/.m2/repository/com/parsely/parsely/*
functional-tests:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: set up JDK
uses: actions/setup-java@v3
with:
java-version: '17'
distribution: 'temurin'
cache: gradle
- name: Functional Tests
uses: reactivecircus/[email protected]
with:
working-directory: .
api-level: 31
profile: Nexus 6
arch: x86_64
force-avd-creation: false
avd-name: macOS-avd-x86_64-31
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
disable-animations: true
script: ./gradlew :parsely:connectedDebugAndroidTest
- name: Publish build artifacts
uses: actions/upload-artifact@v3
if: always()
with:
name: artifact
path: ./parsely/build/reports/*
24 changes: 21 additions & 3 deletions parsely/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,23 @@ plugins {
id 'org.jetbrains.kotlinx.kover'
}

ext {
assertJVersion = '3.24.2'
mockWebServerVersion = '4.12.0'
}

android {
compileSdkVersion 33

defaultConfig {
minSdkVersion 21
targetSdkVersion 33

testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
testInstrumentationRunnerArguments clearPackageData: 'true'
}
testOptions {
execution 'ANDROIDX_TEST_ORCHESTRATOR'
}
buildTypes {
release {
Expand Down Expand Up @@ -52,12 +63,19 @@ dependencies {
implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.3'
implementation 'com.google.android.gms:play-services-ads-identifier:18.0.1'
implementation 'androidx.lifecycle:lifecycle-process:2.6.2'

testImplementation 'org.robolectric:robolectric:4.10.3'
testImplementation 'androidx.test:core:1.5.0'
testImplementation 'org.assertj:assertj-core:3.24.2'
testImplementation "org.assertj:assertj-core:$assertJVersion"
testImplementation 'junit:junit:4.13.2'
testImplementation 'com.squareup.okhttp3:mockwebserver:4.12.0'
testImplementation "com.squareup.okhttp3:mockwebserver:$mockWebServerVersion"

androidTestImplementation 'androidx.test.ext:junit:1.1.5'
androidTestImplementation 'androidx.test:rules:1.5.0'
androidTestImplementation "org.assertj:assertj-core:$assertJVersion"
androidTestImplementation "com.squareup.okhttp3:mockwebserver:$mockWebServerVersion"
androidTestImplementation 'androidx.test:runner:1.5.2'
androidTestUtil 'androidx.test:orchestrator:1.4.2'
}

apply from: "${rootProject.projectDir}/publication.gradle"
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package com.parsely.parselyandroid

import android.app.Activity
import androidx.test.core.app.ActivityScenario
import androidx.test.ext.junit.runners.AndroidJUnit4
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.core.type.TypeReference
import com.fasterxml.jackson.databind.ObjectMapper
import java.io.File
import java.io.FileInputStream
import java.io.ObjectInputStream
import java.lang.reflect.Field
import java.nio.file.Path
import kotlin.io.path.Path
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.seconds
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.coroutines.yield
import okhttp3.mockwebserver.MockResponse
import okhttp3.mockwebserver.MockWebServer
import okhttp3.mockwebserver.RecordedRequest
import org.assertj.core.api.Assertions.assertThat
import org.junit.Test
import org.junit.runner.RunWith


@RunWith(AndroidJUnit4::class)
class FunctionalTests {

private lateinit var parselyTracker: ParselyTracker
private val server = MockWebServer()
private val url = server.url("/").toString()
private lateinit var appsFiles: Path

private fun beforeEach(activity: Activity) {
appsFiles = Path(activity.filesDir.path)

if (File("$appsFiles/$localStorageFileName").exists()) {
throw RuntimeException("Local storage file exists. Something went wrong with orchestrating the tests.")
}
}

/**
* In this scenario, the consumer application tracks more than 50 events-threshold during a flush interval.
* The SDK will save the events to disk and send them in the next flush interval.
* At the end, when all events are sent, the SDK will delete the content of local storage file.
*/
@Test
fun appTracksEventsAboveQueueSizeLimit() {
ActivityScenario.launch(SampleActivity::class.java).use { scenario ->
scenario.onActivity { activity: Activity ->
beforeEach(activity)
server.enqueue(MockResponse().setResponseCode(200))
parselyTracker = initializeTracker(activity)

repeat(51) {
parselyTracker.trackPageview("url", null, null, null)
}
}

// Waits for the SDK to send events (flush interval passes)
val requestPayload = server.takeRequest().toMap()
assertThat(requestPayload["events"]).hasSize(51)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Question (❓): Although I understand how this test is working and what it tried to verify, I feel that something is missing here, an assertion of some kind, something that will showcase what was documented on the test case above:

  • The SDK will save the events to disk...
  • ...and send them in the next flush interval.
  • At the end, when all events are sent...
  • ...the SDK will delete the content of local storage file.

I mean, can we assert that the SDK is saving the events first? For example, will it save all 51 of them, or only the 1 event that exceed the threshold, something else? Then, what happens during the first flush interval, what happens during the next flush interval, these kind of question. Apologies for trying to push things, I am just trying to review this as a very naive test reader that would like to understand this SDK through testing, assuming I don't have any prior knowledge or the SDK. Wdyt, too much? 🤔

PS: I also naively tried to add something basic, the assertThat(locallyStoredEvents).isNotEmpty before the waitFor { locallyStoredEvents.isEmpty() }, to see if that will pass, but it failed. 🤷

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean, can we assert that the SDK is saving the events first?

This is something I pivoted from in 8d4a0d3. IMO, the functional tests shouldn't check for such implementation details. The things we're interested in/testing here are:

  • that SDK has sent the same number of events consumer app tracked (asserted in line 65)
  • that SDK doesn't store additional data (line 69). Actually, we shouldn't even check this probably in a functional test, but this is a safe-check as I know we have some issues with events duplications.

Then, what happens during the first flush interval, what happens during the next flush interval, these kind of question.

There is only one flush, and it's tested 👍 I'll be extending this tests suite to cover more cases, like multiple flushes, but here we test "what would happen if user recorded more than 50 events quickly".

PS: I also naively tried to add something basic, the assertThat(locallyStoredEvents).isNotEmpty before the waitFor { locallyStoredEvents.isEmpty() }, to see if that will pass, but it failed. 🤷

Good point, but it's only timing thing. It seems that assertions above take long enough to give SDK time to clean the file.

Try this:

Subject: [PATCH] refactor: move the class to Kotlin

This class is relatively simple and difficult to test because of `java.utils.Timer`. That's why I decided to make migration to Kotlin right away, without unit tests coverage first.
---
Index: parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt b/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt
--- a/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt	(revision 5900d93eff388bc5b2d98c44072cee07d91b1a96)
+++ b/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt	(date 1698666962291)
@@ -62,11 +62,11 @@
 
             // Waits for the SDK to send events (flush interval passes)
             val requestPayload = server.takeRequest().toMap()
+            assertThat(locallyStoredEvents).isNotEmpty
             assertThat(requestPayload["events"]).hasSize(51)
 
             // Wait a moment to give SDK time to delete the content of local storage file
             waitFor { locallyStoredEvents.isEmpty() }
-            assertThat(locallyStoredEvents).isEmpty()
         }
     }

and the test should fail!

WDYT?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the reply @wzieba ! 👍

This is something I pivoted from in 8d4a0d3. IMO, the functional tests shouldn't check for such implementation details. The things we're interested in/testing here are:

  • that SDK has sent the same number of events consumer app tracked (asserted in line 65)
  • that SDK doesn't store additional data (line 69). Actually, we shouldn't even check this probably in a functional test, but this is a safe-check as I know we have some issues with events duplications.

👍 if you are also 👍 with that, I just wanted to question it a bit... 😊

There is only one flush, and it's tested 👍 I'll be extending this tests suite to cover more cases, like multiple flushes, but here we test "what would happen if user recorded more than 50 events quickly".

Thanks for the clarification on that. 👍

Good point, but it's only timing thing. It seems that assertions above take long enough to give SDK time to clean the file.

Try this:

  • assertThat(locallyStoredEvents).isNotEmpty

and the test should fail!

WDYT?

Yea, it is hard to assert those things... 😭

FYI: The test should fail, I don't understand that, the test should success with this change, is it not? It failing was the actual problem I had as well, as I would have expected it to succeed before waiting... 🤔

PS: Apologies, I am a bit confused it seems... 🤷

  • assertThat(locallyStoredEvents).isEmpty()

Why did you remove this assertion from the patch you shared? 🤔

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, sorry I made some confusion with the diff! Please ignore the diff in previous comment. But it's still case of timing - check this:

Subject: [PATCH] fix: add `intern` to `ROOT_URL` of emulator localhost
---
Index: parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt b/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt
--- a/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt	(revision d2a25dbc864396d67dc517943a5a079cffb86304)
+++ b/parsely/src/androidTest/java/com/parsely/parselyandroid/FunctionalTests.kt	(date 1698668794333)
@@ -71,6 +71,7 @@
     }
 
     private fun RecordedRequest.toMap(): Map<String, List<Event>> {
+        assertThat(locallyStoredEvents).isEmpty()
         val listType: TypeReference<Map<String, List<Event>>> =
             object : TypeReference<Map<String, List<Event>>>() {}
 

In this case the test will fail although, in theory, it shouldn't because after making the request, SDK will clean the local file. But, as the things happen on different threads, we don't have certainty what will happen first - assertion, or file deletion.

In case of this test - the assertions and JSON mapping take enough time, that, at least in theory, we shouldn't need to wait. Yet, relying on "JSON serialization being slow" as a determiner of test success is super flaky. Hence, the waitFor method.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But it's still case of timing...

Thanks for sharing a new diff, and I see, thanks for explaining again! ✅

I don't want to block this PR, feel free to merge it when you think it is ready, I just wanted to question it a bit just to get a better idea on the actual value of this functional test and how you would that be of use later on. 👍

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the questions! We can always iterate over how those tests look now. For now, I think this test brings value - we're sure that saving and getting data from the local file is working fine and doesn't produce duplicates.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 💯 🥇

// Wait a moment to give SDK time to delete the content of local storage file
waitFor { locallyStoredEvents.isEmpty() }
assertThat(locallyStoredEvents).isEmpty()
}
}

private fun RecordedRequest.toMap(): Map<String, List<Event>> {
val listType: TypeReference<Map<String, List<Event>>> =
object : TypeReference<Map<String, List<Event>>>() {}

return ObjectMapper().readValue(body.readUtf8(), listType)
}

@JsonIgnoreProperties(ignoreUnknown = true)
data class Event(
@JsonProperty("idsite") var idsite: String,
)

private val locallyStoredEvents
get() = FileInputStream(File("$appsFiles/$localStorageFileName")).use {
ObjectInputStream(it).use { objectInputStream ->
@Suppress("UNCHECKED_CAST")
objectInputStream.readObject() as ArrayList<Map<String, Any>>
}
}

private fun initializeTracker(activity: Activity): ParselyTracker {
return ParselyTracker.sharedInstance(
siteId, flushInterval.inWholeSeconds.toInt(), activity.application
).apply {
val f: Field = this::class.java.getDeclaredField("ROOT_URL")
f.isAccessible = true
f.set(this, url)
}
}

private companion object {
const val siteId = "123"
const val localStorageFileName = "parsely-events.ser"
val flushInterval = 10.seconds
}

class SampleActivity : Activity()

private fun waitFor(condition: () -> Boolean) = runBlocking {
withTimeoutOrNull(500.milliseconds) {
while (true) {
yield()
if (condition()) {
break
}
}
}
}
}
11 changes: 11 additions & 0 deletions parsely/src/debug/AndroidManifest.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.parsely.parselyandroid">

<uses-permission android:name="android.permission.INTERNET"/>
<application
android:usesCleartextTraffic="true">
<activity android:name=".FunctionalTests$SampleActivity"/>
</application>

</manifest>
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ public class ParselyTracker {
private static final int QUEUE_SIZE_LIMIT = 50;
private static final int STORAGE_SIZE_LIMIT = 100;
private static final String STORAGE_KEY = "parsely-events.ser";
// emulator localhost
// private static final String ROOT_URL = "http://10.0.2.2:5001/";
private static final String ROOT_URL = "https://p1.parsely.com/";
@SuppressWarnings("StringOperationCanBeSimplified")
// private static final String ROOT_URL = "http://10.0.2.2:5001/".intern(); // emulator localhost
private static final String ROOT_URL = "https://p1.parsely.com/".intern();
protected ArrayList<Map<String, Object>> eventQueue;
private boolean isDebug;
private final Context context;
Expand Down