Skip to content

Commit eb96635

Browse files
committed
Fix #4756: Add support for logging an invested engagement event (#4757)
## Explanation Fixes #4756 This PR introduces a new event for tracking individual play sessions where a learner has reached a level of 'invested' engagement, where 'invested' here is considered to be strong engagement with a likelihood of continuing at least with that play session. This metric is planned to be used as one of the team's conversion metrics to better help track the user marketing pipeline by helping to determine how we can better reach learners who are more likely to reach this level of engagement with lessons (and, thus, hopefully learn what they need to). Note that the event is based on a single play session, not a profile or even a single exploration (so if a user pauses and resumes an exploration, the count for engagement resets **from that point**). Engagement means completing _and_ moving past at minimum 3 cards (which may just be simple 'Continue' button interactions). For simplicity, this PR keeps the new event name the same between the Kenya & non-Kenya styles of naming events. I've verified that the event is logging as expected using Firebase's DebugView: ![image](https://user-images.githubusercontent.com/12983742/203501899-dbccd386-4e04-4966-82bc-a4646b3b742c.png) ## Essential Checklist - [x] The PR title and explanation each start with "Fix #bugnum: " (If this PR fixes part of an issue, prefix the title with "Fix part of #bugnum: ...".) - [x] Any changes to [scripts/assets](https://github.com/oppia/oppia-android/tree/develop/scripts/assets) files have their rationale included in the PR explanation. - [x] The PR follows the [style guide](https://github.com/oppia/oppia-android/wiki/Coding-style-guide). - [x] The PR does not contain any unnecessary code changes from Android Studio ([reference](https://github.com/oppia/oppia-android/wiki/Guidance-on-submitting-a-PR#undo-unnecessary-changes)). - [x] The PR is made from a branch that's **not** called "develop" and is up-to-date with "develop". - [x] The PR is **assigned** to the appropriate reviewers ([reference](https://github.com/oppia/oppia-android/wiki/Guidance-on-submitting-a-PR#clarification-regarding-assignees-and-reviewers-section)). ## For UI-specific PRs only This PR is introducing a log and has no impact on the UI or UX of the app.
1 parent 0818246 commit eb96635

File tree

12 files changed

+393
-27
lines changed

12 files changed

+393
-27
lines changed

domain/src/main/java/org/oppia/android/domain/exploration/ExplorationProgressController.kt

+13
Original file line numberDiff line numberDiff line change
@@ -1046,6 +1046,9 @@ class ExplorationProgressController @Inject constructor(
10461046
private var helpIndex = HelpIndex.getDefaultInstance()
10471047
private var availableCardCount: Int = -1
10481048

1049+
private var hasReachedInvestedEngagement = false
1050+
private var completedStateCount = 0
1051+
10491052
/**
10501053
* The [LearnerAnalyticsLogger.ExplorationAnalyticsLogger] to be used for logging
10511054
* exploration-specific events.
@@ -1087,6 +1090,13 @@ class ExplorationProgressController @Inject constructor(
10871090

10881091
// Force the card count to update.
10891092
availableCardCount = explorationProgress.stateDeck.getViewedStateCount()
1093+
1094+
if (!hasReachedInvestedEngagement &&
1095+
completedStateCount >= MINIMUM_COMPLETED_STATE_COUNT_FOR_INVESTED_ENGAGEMENT
1096+
) {
1097+
it.logInvestedEngagement()
1098+
hasReachedInvestedEngagement = true
1099+
}
10901100
}
10911101
}
10921102

@@ -1106,6 +1116,7 @@ class ExplorationProgressController @Inject constructor(
11061116
fun endState() {
11071117
stateAnalyticsLogger?.logEndCard()
11081118
explorationAnalyticsLogger.endCard()
1119+
completedStateCount++
11091120
}
11101121

11111122
/** Checks and logs for hint-based changes based on the provided [HelpIndex]. */
@@ -1279,6 +1290,8 @@ class ExplorationProgressController @Inject constructor(
12791290
}
12801291

12811292
private companion object {
1293+
private const val MINIMUM_COMPLETED_STATE_COUNT_FOR_INVESTED_ENGAGEMENT = 3
1294+
12821295
/**
12831296
* Returns a collectable [Flow] that notifies [collector] for this [StateFlow]s initial state,
12841297
* and every change after.

domain/src/main/java/org/oppia/android/domain/oppialogger/analytics/LearnerAnalyticsLogger.kt

+9
Original file line numberDiff line numberDiff line change
@@ -384,6 +384,15 @@ class LearnerAnalyticsLogger @Inject constructor(
384384
logStateEvent(contentId, ::createPlayVoiceOverContext, EventBuilder::setPlayVoiceOverContext)
385385
}
386386

387+
/**
388+
* Logs that the learner has demonstrated an invested engagement in the lesson (that is, they've
389+
* played far enough in the lesson to indicate that they're not just quickly browsing & then
390+
* leaving).
391+
*/
392+
fun logInvestedEngagement() {
393+
logStateEvent(EventBuilder::setReachInvestedEngagement)
394+
}
395+
387396
private fun logStateEvent(setter: EventBuilder.(ExplorationContext) -> EventBuilder) =
388397
logStateEvent(Unit, { _, context -> context }, setter)
389398

domain/src/test/java/org/oppia/android/domain/exploration/ExplorationProgressControllerTest.kt

+227
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import org.oppia.android.app.model.EphemeralState
2121
import org.oppia.android.app.model.EphemeralState.StateTypeCase.COMPLETED_STATE
2222
import org.oppia.android.app.model.EphemeralState.StateTypeCase.PENDING_STATE
2323
import org.oppia.android.app.model.EphemeralState.StateTypeCase.TERMINAL_STATE
24+
import org.oppia.android.app.model.EventLog.Context.ActivityContextCase.REACH_INVESTED_ENGAGEMENT
2425
import org.oppia.android.app.model.Exploration
2526
import org.oppia.android.app.model.ExplorationCheckpoint
2627
import org.oppia.android.app.model.Fraction
@@ -1972,6 +1973,226 @@ class ExplorationProgressControllerTest {
19721973
}
19731974
}
19741975

1976+
@Test
1977+
fun testPlayNewExp_firstCard_notFinished_doesNotLogReachInvestedEngagementEvent() {
1978+
logIntoAnalyticsReadyAdminProfile()
1979+
1980+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
1981+
waitForGetCurrentStateSuccessfulLoad()
1982+
1983+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
1984+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
1985+
}
1986+
assertThat(hasEngagementEvent).isFalse()
1987+
}
1988+
1989+
@Test
1990+
fun testPlayNewExp_finishFirstCard_moveToSecond_doesNotLogReachInvestedEngagementEvent() {
1991+
logIntoAnalyticsReadyAdminProfile()
1992+
1993+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
1994+
waitForGetCurrentStateSuccessfulLoad()
1995+
playThroughPrototypeState1AndMoveToNextState()
1996+
1997+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
1998+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
1999+
}
2000+
assertThat(hasEngagementEvent).isFalse()
2001+
}
2002+
2003+
@Test
2004+
fun testPlayNewExp_finishThreeCards_doNotProceed_doesNotLogReachInvestedEngagementEvent() {
2005+
logIntoAnalyticsReadyAdminProfile()
2006+
2007+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2008+
waitForGetCurrentStateSuccessfulLoad()
2009+
playThroughPrototypeState1AndMoveToNextState()
2010+
playThroughPrototypeState2AndMoveToNextState()
2011+
submitPrototypeState3Answer()
2012+
2013+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
2014+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2015+
}
2016+
assertThat(hasEngagementEvent).isFalse()
2017+
}
2018+
2019+
@Test
2020+
fun testPlayNewExp_finishThreeCards_moveToFour_logsReachInvestedEngagementEvent() {
2021+
logIntoAnalyticsReadyAdminProfile()
2022+
2023+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2024+
waitForGetCurrentStateSuccessfulLoad()
2025+
playThroughPrototypeState1AndMoveToNextState()
2026+
playThroughPrototypeState2AndMoveToNextState()
2027+
playThroughPrototypeState3AndMoveToNextState()
2028+
2029+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
2030+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2031+
}
2032+
val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
2033+
assertThat(hasEngagementEvent).isTrue()
2034+
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
2035+
hasStateNameThat().isEqualTo("ItemSelectionMinOne")
2036+
}
2037+
}
2038+
2039+
@Test
2040+
fun testPlayNewExp_finishFourCards_moveToFive_logsReachInvestedEngagementEventOnlyOnce() {
2041+
logIntoAnalyticsReadyAdminProfile()
2042+
2043+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2044+
waitForGetCurrentStateSuccessfulLoad()
2045+
playThroughPrototypeState1AndMoveToNextState()
2046+
playThroughPrototypeState2AndMoveToNextState()
2047+
playThroughPrototypeState3AndMoveToNextState()
2048+
playThroughPrototypeState4AndMoveToNextState()
2049+
2050+
// The engagement event should only be logged once during a play session, even if the user
2051+
// continues past that point.
2052+
val engagementEventCount = fakeAnalyticsEventLogger.countEvents {
2053+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2054+
}
2055+
assertThat(engagementEventCount).isEqualTo(1)
2056+
}
2057+
2058+
@Test
2059+
fun testPlayNewExp_firstTwo_startOver_playFirst_doesNotLogReachInvestedEngagementEvent() {
2060+
logIntoAnalyticsReadyAdminProfile()
2061+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2062+
waitForGetCurrentStateSuccessfulLoad()
2063+
playThroughPrototypeState1AndMoveToNextState()
2064+
playThroughPrototypeState2AndMoveToNextState()
2065+
2066+
// Restart the exploration.
2067+
restartExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2068+
waitForGetCurrentStateSuccessfulLoad()
2069+
playThroughPrototypeState1AndMoveToNextState()
2070+
2071+
// No engagement event should be logged, even though 3 total states were completed from the
2072+
// first and second sessions (cumulatively).
2073+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
2074+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2075+
}
2076+
assertThat(hasEngagementEvent).isFalse()
2077+
}
2078+
2079+
@Test
2080+
fun testPlayNewExp_firstTwo_startOver_playThreeAndMove_logsReachInvestedEngagementEvent() {
2081+
logIntoAnalyticsReadyAdminProfile()
2082+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2083+
waitForGetCurrentStateSuccessfulLoad()
2084+
playThroughPrototypeState1AndMoveToNextState()
2085+
playThroughPrototypeState2AndMoveToNextState()
2086+
2087+
// Restart the exploration.
2088+
restartExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2089+
waitForGetCurrentStateSuccessfulLoad()
2090+
playThroughPrototypeState1AndMoveToNextState()
2091+
playThroughPrototypeState2AndMoveToNextState()
2092+
playThroughPrototypeState3AndMoveToNextState()
2093+
2094+
// An engagement event should be logged since the new session uniquely finished 3 states.
2095+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
2096+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2097+
}
2098+
val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
2099+
assertThat(hasEngagementEvent).isTrue()
2100+
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
2101+
hasStateNameThat().isEqualTo("ItemSelectionMinOne")
2102+
}
2103+
}
2104+
2105+
@Test
2106+
fun testResumeExp_stateOneTwoDone_finishThreeAndMoveForward_noLogReachInvestedEngagementEvent() {
2107+
logIntoAnalyticsReadyAdminProfile()
2108+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2109+
waitForGetCurrentStateSuccessfulLoad()
2110+
playThroughPrototypeState1AndMoveToNextState()
2111+
playThroughPrototypeState2AndMoveToNextState()
2112+
2113+
// End, then resume the exploration and complete the third state.
2114+
endExploration()
2115+
val checkPoint = retrieveExplorationCheckpoint(TEST_EXPLORATION_ID_2)
2116+
resumeExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2, checkPoint)
2117+
playThroughPrototypeState3AndMoveToNextState()
2118+
2119+
// Despite the first three states now being completed, this isn't an engagement event since the
2120+
// user hasn't finished three states within *one* session.
2121+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
2122+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2123+
}
2124+
assertThat(hasEngagementEvent).isFalse()
2125+
}
2126+
2127+
@Test
2128+
fun testResumeExp_stateOneTwoDone_finishThreeMoreAndMove_logsReachInvestedEngagementEvent() {
2129+
logIntoAnalyticsReadyAdminProfile()
2130+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2131+
waitForGetCurrentStateSuccessfulLoad()
2132+
playThroughPrototypeState1AndMoveToNextState()
2133+
playThroughPrototypeState2AndMoveToNextState()
2134+
2135+
// End, then resume the exploration and complete the third state.
2136+
endExploration()
2137+
val checkPoint = retrieveExplorationCheckpoint(TEST_EXPLORATION_ID_2)
2138+
resumeExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2, checkPoint)
2139+
playThroughPrototypeState3AndMoveToNextState()
2140+
playThroughPrototypeState4AndMoveToNextState()
2141+
playThroughPrototypeState5AndMoveToNextState()
2142+
2143+
// An engagement event should be logged now since the user completed 3 new states in the current
2144+
// session.
2145+
val hasEngagementEvent = fakeAnalyticsEventLogger.hasEventLogged {
2146+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2147+
}
2148+
val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
2149+
assertThat(hasEngagementEvent).isTrue()
2150+
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
2151+
hasStateNameThat().isEqualTo("NumberInput")
2152+
}
2153+
}
2154+
2155+
@Test
2156+
fun testResumeExp_finishThree_thenAnotherThreeAfterResume_logsInvestedEngagementEventTwice() {
2157+
logIntoAnalyticsReadyAdminProfile()
2158+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2159+
waitForGetCurrentStateSuccessfulLoad()
2160+
playThroughPrototypeState1AndMoveToNextState()
2161+
playThroughPrototypeState2AndMoveToNextState()
2162+
playThroughPrototypeState3AndMoveToNextState()
2163+
2164+
// End, then resume the exploration and complete the third state.
2165+
endExploration()
2166+
val checkPoint = retrieveExplorationCheckpoint(TEST_EXPLORATION_ID_2)
2167+
resumeExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2, checkPoint)
2168+
playThroughPrototypeState4AndMoveToNextState()
2169+
playThroughPrototypeState5AndMoveToNextState()
2170+
playThroughPrototypeState6AndMoveToNextState()
2171+
2172+
// Playing enough states for the engagement event before and after resuming should result in it
2173+
// being logged twice (once for each session).
2174+
val engagementEventCount = fakeAnalyticsEventLogger.countEvents {
2175+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2176+
}
2177+
assertThat(engagementEventCount).isEqualTo(2)
2178+
}
2179+
2180+
@Test
2181+
fun testPlayNewExp_getToEngagementEvent_playOtherExpAndDoSame_logsEngagementEventAgain() {
2182+
logIntoAnalyticsReadyAdminProfile()
2183+
2184+
// Play through the full prototype exploration twice.
2185+
playThroughPrototypeExplorationInNewSession()
2186+
playThroughPrototypeExplorationInNewSession()
2187+
2188+
// Playing through two complete exploration sessions should result in the engagement event being
2189+
// logged twice (once for each session).
2190+
val engagementEventCount = fakeAnalyticsEventLogger.countEvents {
2191+
it.context.activityContextCase == REACH_INVESTED_ENGAGEMENT
2192+
}
2193+
assertThat(engagementEventCount).isEqualTo(2)
2194+
}
2195+
19752196
@Test
19762197
fun testSubmitAnswer_correctAnswer_logsEndCardAndSubmitAnswerEvents() {
19772198
logIntoAnalyticsReadyAdminProfile()
@@ -2325,6 +2546,12 @@ class ExplorationProgressControllerTest {
23252546
return waitForGetCurrentStateSuccessfulLoad()
23262547
}
23272548

2549+
private fun playThroughPrototypeExplorationInNewSession() {
2550+
startPlayingNewExploration(TEST_TOPIC_ID_0, TEST_STORY_ID_0, TEST_EXPLORATION_ID_2)
2551+
playThroughPrototypeExploration()
2552+
endExploration()
2553+
}
2554+
23282555
private fun playThroughPrototypeExploration(): EphemeralState {
23292556
playThroughPrototypeState1AndMoveToNextState()
23302557
playThroughPrototypeState2AndMoveToNextState()

domain/src/test/java/org/oppia/android/domain/oppialogger/analytics/LearnerAnalyticsLoggerTest.kt

+24
Original file line numberDiff line numberDiff line change
@@ -1400,6 +1400,30 @@ class LearnerAnalyticsLoggerTest {
14001400
assertThat(log.type).isEqualTo(Log.ERROR)
14011401
}
14021402

1403+
@Test
1404+
fun testStateAnalyticsLogger_logReachInvestedEngagement_logsStateEventWithStateName() {
1405+
val exploration5 = loadExploration(TEST_EXPLORATION_ID_5)
1406+
val expLogger = learnerAnalyticsLogger.beginExploration(exploration5)
1407+
val stateLogger = expLogger.startCard(exploration5.getStateByName(TEST_EXP_5_STATE_THREE_NAME))
1408+
1409+
stateLogger.logInvestedEngagement()
1410+
1411+
val eventLog = fakeAnalyticsEventLogger.getMostRecentEvent()
1412+
assertThat(eventLog).isEssentialPriority()
1413+
assertThat(eventLog).hasReachedInvestedEngagementContextThat {
1414+
hasTopicIdThat().isEqualTo(TEST_TOPIC_ID)
1415+
hasStoryIdThat().isEqualTo(TEST_STORY_ID)
1416+
hasExplorationIdThat().isEqualTo(TEST_EXPLORATION_ID_5)
1417+
hasSessionIdThat().isEqualTo(DEFAULT_INITIAL_SESSION_ID)
1418+
hasVersionThat().isEqualTo(5)
1419+
hasStateNameThat().isEqualTo(TEST_EXP_5_STATE_THREE_NAME)
1420+
hasLearnerDetailsThat {
1421+
hasLearnerIdThat().isEqualTo(TEST_LEARNER_ID)
1422+
hasInstallationIdThat().isEqualTo(TEST_INSTALL_ID)
1423+
}
1424+
}
1425+
}
1426+
14031427
private fun loadExploration(expId: String): Exploration {
14041428
return monitorFactory.waitForNextSuccessfulResult(
14051429
explorationDataController.getExplorationById(profileId, expId)

model/src/main/proto/oppia_logger.proto

+4
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,10 @@ message EventLog {
123123
// value here has no importance and is always 'true'.
124124
bool open_profile_chooser = 32;
125125

126+
// The event being logged indicates that the user has reached an invested level of learning
127+
// engagement in a lesson.
128+
ExplorationContext reach_invested_engagement = 34;
129+
126130
// Indicates that something went wrong when trying to log a learner analytics even for the
127131
// device corresponding to the specified device ID.
128132
string install_id_for_failed_analytics_log = 33;

testing/src/main/java/org/oppia/android/testing/FakeAnalyticsEventLogger.kt

+5-2
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ class FakeAnalyticsEventLogger @Inject constructor() : AnalyticsEventLogger {
2727
/** Clears all the events that are currently logged. */
2828
fun clearAllEvents() = eventList.clear()
2929

30-
/** Checks if a certain event has been logged or not. */
31-
fun hasEventLogged(eventLog: EventLog): Boolean = eventList.contains(eventLog)
30+
/** Returns whether a certain event has been logged or not, based on the provided [predicate]. */
31+
fun hasEventLogged(predicate: (EventLog) -> Boolean): Boolean = eventList.find(predicate) != null
32+
33+
/** Returns the number of logged events that match the provided [predicate]. */
34+
fun countEvents(predicate: (EventLog) -> Boolean): Int = eventList.count(predicate)
3235

3336
/** Returns true if there are no events logged. */
3437
fun noEventsPresent(): Boolean = eventList.isEmpty()

0 commit comments

Comments
 (0)