diff --git a/.github/workflows/gradle.yml b/.github/workflows/build.yml similarity index 89% rename from .github/workflows/gradle.yml rename to .github/workflows/build.yml index 907c2f04..3a708a09 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,12 @@ jobs: - name: Test with Gradle Wrapper run: ./gradlew test + - name: Build simple-console example + run: | + ./gradlew publishToMavenLocal + cd examples/simple-console + ./gradlew build + - name: Upload report if failed if: ${{ failure() }} uses: actions/upload-artifact@v2 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000..c697bad1 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,30 @@ +name: Documentation + +on: + push: + branches: [ master ] + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@v1 + + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 8 + + - name: Build + run: ./gradlew build + + - name: Deploy to GitHub Pages + if: success() + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./build/docs/javadoc diff --git a/CHANGELOG b/CHANGELOG index c5c176e4..e55776f2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,20 @@ +Java 0.12.0 (2022-03-24) +----------------------- +Choose HTTP response codes not to retry (#316) +Add Javadoc generation (#137) +Deprecate SimpleEmitter (#309) +Update junit and jackson-databind (#294) +Update copyright notices to 2022 (#312) +Return eventId from Tracker.track() (#304) +Add retry to in-memory storage system (#156) +Rename bufferSize to batchSize (#306) +Add benchmarking tests (#300) +Update simple-console example (#295) +Provide method for stopping Tracker executorService (#297) +Refactor TrackerEvents for event payload creation (#291) +Extract event storage from Emitter (#290) +Attribute community contributions in changelog (#289) + Java 0.11.0 (2021-12-14) ----------------------- Remove logging of user supplied values (#286) diff --git a/LICENSE b/LICENSE index b2d6fe1e..e0977a71 100644 --- a/LICENSE +++ b/LICENSE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2021 Snowplow Analytics Ltd. + Copyright 2022 Snowplow Analytics Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -199,4 +199,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/README.md b/README.md index 1a4ba234..4a9191fc 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,10 @@ With this tracker you can collect event data from your Java-based desktop and se ## Find out more -| Snowplow Docs | Contributing | -|---------------------------------|-----------------------------------| -| ![i1][techdocs-image] | ![i4][contributing-image] | -| **[Snowplow Docs][techdocs]** | **[Contributing](CONTRIBUTING.md)** | +| Snowplow Docs | API Docs | Contributing | +|-------------------------------|-----------|-----------------------------------| +| ![i1][techdocs-image] | ![i1][techdocs-image] | ![i4][contributing-image] | +| **[Snowplow Docs][techdocs]** | **[Javadoc Docs][apidocs]** | **[Contributing](CONTRIBUTING.md)** | ## Maintainer Quickstart @@ -33,15 +33,22 @@ To run the tests using your installed JDK, run: $ ./gradlew build ``` -We have also included a simple demo, found in the `examples/simple-console` folder. You will need a JDK installed to run it. When run, it sends several events to your event collector. +We have also included a simple demo, found in the `examples/simple-console` folder. You will need a JDK installed to run it. When run, it sends several events to your event collector. For a simple event collector, we advise using the [Snowplow Micro][micro] testing pipeline. +To run simple-console using the current Maven Central version of the Java tracker: +```bash +$ cd examples/simple-console +$ ./gradlew jar +$ java -jar ./build/libs/simple-console-all-0.0.1.jar "http://" +``` + +To run simple-console using a local version of the Java tracker: ```bash $ ./gradlew publishToMavenLocal $ cd examples/simple-console $ ./gradlew jar $ java -jar ./build/libs/simple-console-all-0.0.1.jar "http://" ``` -For a simple event collector, we advise using the [Snowplow Micro][micro] testing pipeline. ## Copyright and license @@ -78,6 +85,7 @@ limitations under the License. [contributing-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/contributing.png [techdocs]: https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/java-tracker/ +[apidocs]: https://snowplow.github.io/snowplow-java-tracker/index.html?overview-summary.html [tracker-classification]: https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/tracker-maintenance-classification/ [early-release]: https://img.shields.io/static/v1?style=flat&label=Snowplow&message=Early%20Release&color=014477&labelColor=9ba0aa&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAMAAAAoLQ9TAAAAeFBMVEVMaXGXANeYANeXANZbAJmXANeUANSQAM+XANeMAMpaAJhZAJeZANiXANaXANaOAM2WANVnAKWXANZ9ALtmAKVaAJmXANZaAJlXAJZdAJxaAJlZAJdbAJlbAJmQAM+UANKZANhhAJ+EAL+BAL9oAKZnAKVjAKF1ALNBd8J1AAAAKHRSTlMAa1hWXyteBTQJIEwRgUh2JjJon21wcBgNfmc+JlOBQjwezWF2l5dXzkW3/wAAAHpJREFUeNokhQOCA1EAxTL85hi7dXv/E5YPCYBq5DeN4pcqV1XbtW/xTVMIMAZE0cBHEaZhBmIQwCFofeprPUHqjmD/+7peztd62dWQRkvrQayXkn01f/gWp2CrxfjY7rcZ5V7DEMDQgmEozFpZqLUYDsNwOqbnMLwPAJEwCopZxKttAAAAAElFTkSuQmCC diff --git a/build.gradle b/build.gradle index 6a1eb761..05cb8863 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -25,7 +25,7 @@ wrapper.gradleVersion = '6.5.0' group = 'com.snowplowanalytics' archivesBaseName = 'snowplow-java-tracker' -version = '0.11.0' +version = '0.12.0' sourceCompatibility = '1.8' targetCompatibility = '1.8' @@ -73,16 +73,16 @@ dependencies { testImplementation 'org.slf4j:slf4j-simple:1.7.30' // Jackson JSON processor - api 'com.fasterxml.jackson.core:jackson-databind:2.11.0' + api 'com.fasterxml.jackson.core:jackson-databind:2.13.1' // Preconditions api 'com.google.guava:guava:31.0-jre' // Testing libraries testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' + testCompileOnly 'junit:junit:4.13.2' + testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' - testImplementation 'org.hamcrest:hamcrest:2.2' testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.2' } @@ -98,7 +98,7 @@ task generateSources { srcFile.parentFile.mkdirs() srcFile.write( """/* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -111,6 +111,9 @@ task generateSources { */ package com.snowplowanalytics.snowplow.tracker; // DO NOT EDIT. AUTO-GENERATED. +/** +* The release version of the Snowplow Java tracker. +*/ public class Version { static final String TRACKER = "java-$project.version"; static final String VERSION = "$project.version"; @@ -218,3 +221,4 @@ signing { println 'Used useInMemoryPgpKeys()' } } + diff --git a/examples/benchmarking/BenchmarkingREADME.md b/examples/benchmarking/BenchmarkingREADME.md new file mode 100644 index 00000000..7e0ff9e8 --- /dev/null +++ b/examples/benchmarking/BenchmarkingREADME.md @@ -0,0 +1,24 @@ +## Benchmarking results + +This benchmarking module is provided for maintainers, allowing them to check that their changes have not degraded performance. It uses the Java microbenchmarking harness, JMH. + +The benchmark test measures the time taken to track one event. Note that this does not include the time for the event to be processed and sent, which happens asynchronously. + +To run the test, navigate to this folder and run: + +```bash +$ ./gradlew build +$ ./gradlew jmh +``` + +The tracker version is set in the `build.gradle` file. Change the specified version to benchmark a different tracker version. +```groovy +dependencies { + jmh 'com.snowplowanalytics:snowplow-java-tracker:0.11.0' +} +``` +Note that you may also need to edit the `TrackerBenchmark` `closeThreads()` code. Versions from 0.12.0 onwards must call a different method. This is explained in in-line comments. + +### Results +See this PR for discussion of benchmarking results: https://github.com/snowplow/snowplow-java-tracker/pull/301 + diff --git a/examples/benchmarking/build.gradle b/examples/benchmarking/build.gradle new file mode 100644 index 00000000..713d8895 --- /dev/null +++ b/examples/benchmarking/build.gradle @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ + +import org.gradle.api.tasks.JavaExec + +plugins { + id 'java' + id "me.champeau.jmh" version "0.6.6" +} + +group 'com.snowplowanalytics' +version '1.0' + +repositories { + mavenLocal { + content { + includeGroup "com.snowplowanalytics" + } + } + mavenCentral() +} + + +dependencies { + jmh 'com.snowplowanalytics:snowplow-java-tracker:0.10.1' +} diff --git a/examples/benchmarking/gradle/wrapper/gradle-wrapper.jar b/examples/benchmarking/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..7454180f Binary files /dev/null and b/examples/benchmarking/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/benchmarking/gradle/wrapper/gradle-wrapper.properties b/examples/benchmarking/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..69a97150 --- /dev/null +++ b/examples/benchmarking/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/examples/benchmarking/gradlew b/examples/benchmarking/gradlew new file mode 100755 index 00000000..744e882e --- /dev/null +++ b/examples/benchmarking/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MSYS* | MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/examples/benchmarking/gradlew.bat b/examples/benchmarking/gradlew.bat new file mode 100644 index 00000000..ac1b06f9 --- /dev/null +++ b/examples/benchmarking/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/examples/benchmarking/settings.gradle b/examples/benchmarking/settings.gradle new file mode 100644 index 00000000..9efe9f92 --- /dev/null +++ b/examples/benchmarking/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'benchmarking' + diff --git a/examples/benchmarking/src/jmh/java/com/snowplowanalytics/TrackerBenchmark.java b/examples/benchmarking/src/jmh/java/com/snowplowanalytics/TrackerBenchmark.java new file mode 100644 index 00000000..460ed692 --- /dev/null +++ b/examples/benchmarking/src/jmh/java/com/snowplowanalytics/TrackerBenchmark.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics; + +import com.snowplowanalytics.snowplow.tracker.Tracker; +import com.snowplowanalytics.snowplow.tracker.emitter.BatchEmitter; +import com.snowplowanalytics.snowplow.tracker.emitter.Emitter; +import com.snowplowanalytics.snowplow.tracker.events.PageView; +import com.snowplowanalytics.snowplow.tracker.http.HttpClientAdapter; +import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.concurrent.TimeUnit; + + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 15, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 20, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Fork(5) +public class TrackerBenchmark { + public static class MockHttpClientAdapter implements HttpClientAdapter { + @Override + public int post(SelfDescribingJson payload) { + return 200; + } + + @Override + public int get(TrackerPayload payload) { + return 0; + } + + @Override + public String getUrl() { + return null; + } + + @Override + public Object getHttpClient() { + return null; + } + } + + public static BatchEmitter getEmitter() { + MockHttpClientAdapter mockHttpClientAdapter = new MockHttpClientAdapter(); + return BatchEmitter.builder() + .httpClientAdapter(mockHttpClientAdapter) + .build(); + } + + public static Tracker getTracker(Emitter emitter) { + return new Tracker.TrackerBuilder(emitter, "namespace", "appId").build(); + } + + public static void closeThreads(Tracker tracker) { + BatchEmitter emitter = (BatchEmitter) tracker.getEmitter(); + emitter.close(); + } + + // This State class exists only to print out the tracker version + @State(Scope.Benchmark) + public static class TrackerVersion { + BatchEmitter emitter = getEmitter(); + Tracker tracker = getTracker(emitter); + + @Setup(Level.Trial) + public void printTrackerVersion() { + System.out.println("Using tracker version: " + tracker.getTrackerVersion()); + } + + @TearDown(Level.Trial) + public void doTearDown() { + System.out.println("Do TearDown for trackerVersion state"); + closeThreads(tracker); + } + } + + // This class creates the tracker components. + // They are recreated for every iteration of the benchmark test. + @State(Scope.Benchmark) + public static class TrackerComponents { + Tracker tracker; + BatchEmitter emitter; + + PageView pageViewEvent = PageView.builder() + .pageUrl("url") + .pageTitle("title") + .referrer("referrer") + .build(); + + @Setup(Level.Iteration) + public void doSetUp() { + emitter = getEmitter(); + tracker = getTracker(emitter); + } + + @TearDown(Level.Iteration) + public void doTearDown() { + closeThreads(tracker); + } + } + + // The Blackhole forces JMH to measure the method. + @Benchmark + public void testTrackEvent(Blackhole blackhole, TrackerComponents trackerComponents, TrackerVersion trackerVersion) { + trackerComponents.tracker.track(trackerComponents.pageViewEvent); + blackhole.consume(trackerComponents); + } +} diff --git a/examples/simple-console/build.gradle b/examples/simple-console/build.gradle index 0497ec92..97217503 100644 --- a/examples/simple-console/build.gradle +++ b/examples/simple-console/build.gradle @@ -6,9 +6,6 @@ targetCompatibility = '1.8' repositories { mavenLocal() - maven { - url "https://snowplow.bintray.com/snowplow-maven" - } mavenCentral() } @@ -19,11 +16,11 @@ test { } dependencies { - implementation 'com.snowplowanalytics:snowplow-java-tracker:0.10.1' + implementation 'com.snowplowanalytics:snowplow-java-tracker:0.+' - implementation ('com.snowplowanalytics:snowplow-java-tracker:0.10.1') { + implementation ('com.snowplowanalytics:snowplow-java-tracker:0.+') { capabilities { - requireCapability 'com.snowplowanalytics:snowplow-java-tracker-okhttp-support:0.10.1' + requireCapability 'com.snowplowanalytics:snowplow-java-tracker-okhttp-support' } } diff --git a/examples/simple-console/gradlew b/examples/simple-console/gradlew index 1b6c7873..f887d101 100755 --- a/examples/simple-console/gradlew +++ b/examples/simple-console/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015-2022 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/examples/simple-console/src/main/java/com/snowplowanalytics/Main.java b/examples/simple-console/src/main/java/com/snowplowanalytics/Main.java index 6b9f2ab3..60e6cc3d 100644 --- a/examples/simple-console/src/main/java/com/snowplowanalytics/Main.java +++ b/examples/simple-console/src/main/java/com/snowplowanalytics/Main.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -14,18 +14,13 @@ package com.snowplowanalytics; import com.snowplowanalytics.snowplow.tracker.DevicePlatform; +import com.snowplowanalytics.snowplow.tracker.Subject; import com.snowplowanalytics.snowplow.tracker.Tracker; import com.snowplowanalytics.snowplow.tracker.emitter.BatchEmitter; -import com.snowplowanalytics.snowplow.tracker.emitter.Emitter; -import com.snowplowanalytics.snowplow.tracker.emitter.RequestCallback; import com.snowplowanalytics.snowplow.tracker.events.*; import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; import java.util.List; -import java.util.Set; -import java.util.HashSet; -import java.util.concurrent.TimeUnit; import static java.util.Collections.singletonList; import com.google.common.collect.ImmutableMap; @@ -39,12 +34,9 @@ public static String getUrlFromArgs(String[] args) { return args[0]; } - public static void main(String[] args) { - Set failedEventIds = new HashSet(); + public static void main(String[] args) throws InterruptedException { String collectorEndpoint = getUrlFromArgs(args); - System.out.println("Sending events to " + collectorEndpoint); - // the application id to attach to events String appId = "java-tracker-sample-console-app"; // the namespace to attach to events @@ -53,20 +45,7 @@ public static void main(String[] args) { // build an emitter, this is used by the tracker to batch and schedule transmission of events BatchEmitter emitter = BatchEmitter.builder() .url(collectorEndpoint) - .requestCallback(new RequestCallback() { - // let us know on successes (may be called multiple times) - @Override - public synchronized void onSuccess(int successCount) { - System.out.println("Successfully sent " + successCount + " events"); - } - - // let us know if something has gone wrong (may be called multiple times) - @Override - public synchronized void onFailure(int successCount, List failedEvents) { - System.err.println("Successfully sent " + successCount + " events; failed to send " + failedEvents.size() + " events"); - } - }) - .bufferSize(4) // send an event every time one is given (no batching). In production this number should be higher, depending on the size/event volume + .batchSize(4) // send batches of 4 events. In production this number should be higher, depending on the size/event volume .build(); // now we have the emitter, we need a tracker to turn our events into something a Snowplow collector can understand @@ -75,22 +54,32 @@ public synchronized void onFailure(int successCount, List failedEvents) { .platform(DevicePlatform.ServerSideApp) .build(); - // This is an example of a custom context - List contexts = singletonList( + System.out.println("Sending events to " + collectorEndpoint); + System.out.println("Using tracker version " + tracker.getTrackerVersion()); + + // This is an example of a custom context entity + List context = singletonList( new SelfDescribingJson( "iglu:com.snowplowanalytics.iglu/anything-c/jsonschema/1-0-0", ImmutableMap.of("foo", "bar"))); - // This is a sample page view event, many other event types (such as self-describing events) are available + // This is an example of a eventSubject for adding user data + Subject eventSubject = new Subject.SubjectBuilder().build(); + eventSubject.setUserId("example@snowplowanalytics.com"); + eventSubject.setLanguage("EN"); + + // This is a sample page view event + // the eventSubject has been included in this event PageView pageViewEvent = PageView.builder() .pageTitle("Snowplow Analytics") .pageUrl("https://www.snowplowanalytics.com") .referrer("https://www.google.com") - .customContext(contexts) + .customContext(context) + .subject(eventSubject) .build(); - tracker.track(pageViewEvent); // the .track method schedules the event for delivery to Snowplow - + // EcommerceTransactionItems are tracked as part of an EcommerceTransaction event + // They are processed into separate events during the `track()` call EcommerceTransactionItem item = EcommerceTransactionItem.builder() .itemId("order_id") .sku("sku") @@ -99,9 +88,10 @@ public synchronized void onFailure(int successCount, List failedEvents) { .name("name") .category("category") .currency("currency") - .customContext(contexts) + .customContext(context) .build(); + // EcommerceTransaction event EcommerceTransaction ecommerceTransaction = EcommerceTransaction.builder() .orderId("order_id") .totalValue(1.0) @@ -112,31 +102,30 @@ public synchronized void onFailure(int successCount, List failedEvents) { .state("state") .country("country") .currency("currency") - .items(item) // EcommerceTransactionItem events are added to a parent EcommerceTransaction - .customContext(contexts) + .items(item) // EcommerceTransactionItem events are added to a parent EcommerceTransaction here + .customContext(context) .build(); - tracker.track(ecommerceTransaction); // This will track two events - // This is an example of a custom "Unsutrcutred" event based on a schema + // This is an example of a custom "Unstructured" event based on a schema + // Unstructured events are also called "self-describing" events + // because of their SelfDescribingJson base Unstructured unstructured = Unstructured.builder() .eventData(new SelfDescribingJson( "iglu:com.snowplowanalytics.iglu/anything-a/jsonschema/1-0-0", ImmutableMap.of("foo", "bar") )) - .customContext(contexts) + .customContext(context) .build(); - tracker.track(unstructured); // This is an example of a ScreenView event which will be translated into an Unstructured event ScreenView screenView = ScreenView.builder() .name("name") .id("id") - .customContext(contexts) + .customContext(context) .build(); - tracker.track(screenView); // This is an example of a Timing event which will be translated into an Unstructured event Timing timing = Timing.builder() @@ -144,14 +133,31 @@ public synchronized void onFailure(int successCount, List failedEvents) { .label("label") .variable("variable") .timing(10) - .customContext(contexts) + .customContext(context) .build(); + // This is an example of a Structured event + Structured structured = Structured.builder() + .category("category") + .action("action") + .label("label") + .property("property") + .value(12.34) + .customContext(context) + .build(); + + tracker.track(pageViewEvent); // the .track method schedules the event for delivery to Snowplow + tracker.track(ecommerceTransaction); // This will track two events + tracker.track(unstructured); + tracker.track(screenView); tracker.track(timing); + tracker.track(structured); // Will close all threads and force send remaining events - // should be 1 left to flush, as we send 5 events with a bufferSize of 4 emitter.close(); + Thread.sleep(5000); + + System.out.println("Tracked 7 events"); } } diff --git a/examples/simple-console/src/test/java/com/snowplowanalytics/MainTest.java b/examples/simple-console/src/test/java/com/snowplowanalytics/MainTest.java index 30698085..01d5720d 100644 --- a/examples/simple-console/src/test/java/com/snowplowanalytics/MainTest.java +++ b/examples/simple-console/src/test/java/com/snowplowanalytics/MainTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -37,4 +37,4 @@ public void testGetUrlEmpty() { Main.getUrlFromArgs(new String[]{}); } -} \ No newline at end of file +} diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/DevicePlatform.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/DevicePlatform.java index b457bfb9..f8578736 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/DevicePlatform.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/DevicePlatform.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -13,6 +13,9 @@ package com.snowplowanalytics.snowplow.tracker; +/** + * The supported platform options for Tracker objects. + */ public enum DevicePlatform { Web { public String toString() { diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/Subject.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/Subject.java index c61958e8..d058c847 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/Subject.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/Subject.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -21,6 +21,8 @@ /** * An object for managing extra event decoration. + * All the properties are optional. However, the timezone is set by default, + * to that of the server. */ public class Subject { @@ -113,6 +115,8 @@ public SubjectBuilder colorDepth(int depth) { } /** + * Note that timezone is set by default to the server's timezone + * (`TimeZone tz = Calendar.getInstance().getTimeZone().getID()`) * @param timezone a timezone string * @return itself */ @@ -234,7 +238,8 @@ public void setColorDepth(int depth) { } /** - * Sets the timezone parameter + * Sets the timezone parameter. Note that timezone is set by default to the server's timezone + * (`TimeZone tz = Calendar.getInstance().getTimeZone().getID()`); * * @param timezone a timezone string */ diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/Tracker.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/Tracker.java index a992c48c..105593a2 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/Tracker.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/Tracker.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -14,11 +14,19 @@ import com.google.common.base.Preconditions; +import com.snowplowanalytics.snowplow.tracker.constants.Constants; +import com.snowplowanalytics.snowplow.tracker.constants.Parameter; import com.snowplowanalytics.snowplow.tracker.emitter.Emitter; import com.snowplowanalytics.snowplow.tracker.events.*; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerEvent; +import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; import com.snowplowanalytics.snowplow.tracker.payload.TrackerParameters; +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; +import java.util.*; + +/** + * Allows tracking of Events. + */ public class Tracker { private Emitter emitter; @@ -42,6 +50,7 @@ private Tracker(TrackerBuilder builder) { this.parameters = new TrackerParameters(builder.appId, builder.platform, builder.namespace, Version.TRACKER, builder.base64Encoded); this.emitter = builder.emitter; this.subject = builder.subject; + } /** @@ -77,6 +86,8 @@ public TrackerBuilder subject(Subject subject) { } /** + * The {@link DevicePlatform} the tracker is running on (default is "srv", ServerSideApp). + * * @param platform The device platform the tracker is running on * @return itself */ @@ -86,7 +97,9 @@ public TrackerBuilder platform(DevicePlatform platform) { } /** - * @param base64 Whether JSONs in the payload should be base-64 encoded + * Whether JSONs in the payload should be base-64 encoded (default is true) + * + * @param base64 JSONs should be encoded or not * @return itself */ public TrackerBuilder base64(Boolean base64) { @@ -107,6 +120,8 @@ public Tracker build() { // --- Setters /** + * Change the Emitter used to send events. + * * @param emitter a new emitter */ public void setEmitter(Emitter emitter) { @@ -133,14 +148,16 @@ public Emitter getEmitter() { } /** - * @return the Tracker Subject + * @return the Tracker-associated Subject */ public Subject getSubject() { return this.subject; } /** - * @return the tracker version that was set + * The Java tracker release version, e.g. 0.12.0. + * + * @return the tracker version */ public String getTrackerVersion() { return this.parameters.getTrackerVersion(); @@ -154,7 +171,7 @@ public String getNamespace() { } /** - * @return the trackers set Application ID + * @return the tracker Application ID */ public String getAppId() { return this.parameters.getAppId(); @@ -168,7 +185,7 @@ public boolean getBase64Encoded() { } /** - * @return the Tracker platform + * @return the Tracker platform, e.g. "srv" */ public DevicePlatform getPlatform() { return this.parameters.getPlatform(); @@ -178,19 +195,131 @@ public DevicePlatform getPlatform() { * @return the wrapper containing the Tracker parameters */ public TrackerParameters getParameters() { - return this.parameters; + return parameters; } // --- Event Tracking Functions /** - * Handles tracking the different types of events that - * the Tracker can encounter. + * Handles tracking the different types of events. + * + * A TrackerPayload object - or more than one, in the case of eCommerceTransaction events - + * will be created from the Event. This is passed to the configured Emitter. + * If the event was successfully added to the Emitter buffer for sending, + * a list containing the payload's eventId string (a UUID) is returned. + * EcommerceTransactions will return all the relevant eventIds in the list. + * If the Emitter event buffer is full, the payload will be lost. In this case, this method + * returns a list containing null. + *

+ * Implementation note: As a side effect of adding a payload to the Emitter, + * it triggers an Emitter thread to emit a batch of events. * * @param event the event to track + * @return a list of eventIDs (UUIDs) */ - public void track(Event event) { - // Emit the event - this.emitter.emit(new TrackerEvent(event, this.parameters, this.subject)); + public List track(Event event) { + List results = new ArrayList<>(); + // a list because Ecommerce events become multiple Payloads + List processedEvents = eventTypeSpecificPreProcessing(event); + for (Event processedEvent : processedEvents) { + // Event ID (eid) and device_created_timestamp (dtm) are generated now when + // the TrackerPayload is created + TrackerPayload payload = (TrackerPayload) processedEvent.getPayload(); + + addTrackerParameters(payload); + addContext(processedEvent, payload); + addSubject(processedEvent, payload); + + boolean addedToBuffer = emitter.add(payload); + if (addedToBuffer) { + results.add(payload.getEventId()); + } else { + results.add(null); + } + } + return results; } + + private List eventTypeSpecificPreProcessing(Event event) { + // Different event types must be processed in slightly different ways. + // EcommerceTransaction events are an outlier, as they are processed into + // multiple payloads (a "tr" event plus one "ti" event per item). + // Because of this, this method returns a list of Events. + List eventList = new ArrayList<>(); + final Class eventClass = event.getClass(); + + if (eventClass.equals(Unstructured.class)) { + // Need to set the Base64 rule for Unstructured events + final Unstructured unstructured = (Unstructured) event; + unstructured.setBase64Encode(parameters.getBase64Encoded()); + eventList.add(unstructured); + + } else if (eventClass.equals(EcommerceTransaction.class)) { + final EcommerceTransaction ecommerceTransaction = (EcommerceTransaction) event; + eventList.add(ecommerceTransaction); + + // Track each item individually + eventList.addAll(ecommerceTransaction.getItems()); + + } else if (eventClass.equals(Timing.class) || eventClass.equals(ScreenView.class)) { + // Timing and ScreenView events are wrapper classes for Unstructured events + // Need to create Unstructured events from them to send. + final Unstructured unstructured = Unstructured.builder() + .eventData((SelfDescribingJson) event.getPayload()) + .customContext(event.getContext()) + .trueTimestamp(event.getTrueTimestamp()) + .subject(event.getSubject()) + .build(); + + unstructured.setBase64Encode(parameters.getBase64Encoded()); + eventList.add(unstructured); + + } else { + eventList.add(event); + } + return eventList; + } + + private void addTrackerParameters(TrackerPayload payload) { + payload.add(Parameter.PLATFORM, parameters.getPlatform().toString()); + payload.add(Parameter.APP_ID, parameters.getAppId()); + payload.add(Parameter.NAMESPACE, parameters.getNamespace()); + payload.add(Parameter.TRACKER_VERSION, parameters.getTrackerVersion()); + } + + private void addContext(Event event, TrackerPayload payload) { + List entities = event.getContext(); + + // Build the final context and add it to the payload + if (entities != null && entities.size() > 0) { + SelfDescribingJson envelope = getFinalContext(entities); + payload.addMap(envelope.getMap(), parameters.getBase64Encoded(), Parameter.CONTEXT_ENCODED, Parameter.CONTEXT); + } + } + + /** + * Builds the final event context. + * + * @param entities the base event context + * @return the final event context json with many entities inside + */ + private SelfDescribingJson getFinalContext(List entities) { + List> entityMaps = new LinkedList<>(); + for (SelfDescribingJson selfDescribingJson : entities) { + entityMaps.add(selfDescribingJson.getMap()); + } + return new SelfDescribingJson(Constants.SCHEMA_CONTEXTS, entityMaps); + } + + private void addSubject(Event event, TrackerPayload payload) { + Subject eventSubject = event.getSubject(); + + // Add subject if available + if (eventSubject != null) { + payload.addMap(new HashMap<>(eventSubject.getSubject())); + } else if (subject != null) { + payload.addMap(new HashMap<>(subject.getSubject())); + } + } + } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/Utils.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/Utils.java index a031e0d3..6bcafe8e 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/Utils.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/Utils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Constants.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Constants.java index 0bf291b4..514dafc1 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Constants.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Constants.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -13,7 +13,7 @@ package com.snowplowanalytics.snowplow.tracker.constants; /** - * Constants which apply to schemas, event types + * Constants that apply to schemas, event types * and sending protocols. */ public class Constants { diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Parameter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Parameter.java index f00cf1a3..abc01458 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Parameter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/constants/Parameter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -13,6 +13,10 @@ package com.snowplowanalytics.snowplow.tracker.constants; +/** + * More constants that define the event properties, which apply to schemas, event types + * and sending protocols. + */ public class Parameter { // General public static final String SCHEMA = "schema"; diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/AbstractEmitter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/AbstractEmitter.java index b1108a4d..5dea6b16 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/AbstractEmitter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/AbstractEmitter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -13,8 +13,8 @@ package com.snowplowanalytics.snowplow.tracker.emitter; import java.util.List; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; @@ -22,43 +22,44 @@ import com.snowplowanalytics.snowplow.tracker.http.HttpClientAdapter; import com.snowplowanalytics.snowplow.tracker.http.OkHttpClientAdapter; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerEvent; +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; import okhttp3.OkHttpClient; /** * AbstractEmitter class which contains common elements to * the emitters wrapped in a builder format. + * Note that SimpleEmitter has been deprecated. */ public abstract class AbstractEmitter implements Emitter { protected HttpClientAdapter httpClientAdapter; - protected RequestCallback requestCallback; - protected ExecutorService executor; + protected ScheduledExecutorService executor; public static abstract class Builder> { private HttpClientAdapter httpClientAdapter; // Optional - private RequestCallback requestCallback = null; // Optional private int threadCount = 50; // Optional - private ExecutorService requestExecutorService = null; // Optional + private ScheduledExecutorService requestExecutorService = null; // Optional private String collectorUrl = null; // Required if not specifying a httpClientAdapter protected abstract T self(); /** - * Set a custom ExecutorService to send http request. + * Set a custom ScheduledExecutorService to send http requests (default is ScheduledThreadPoolExecutor). + *

+ * Implementation note: Be aware that calling `close()` on a BatchEmitter instance + * has a side-effect and will shutdown that ExecutorService. * - * /!\ Be aware that calling `close()` on a BatchEmitter instance has a side-effect and will shutdown that ExecutorService. - * @param executorService the ExecutorService to use + * @param executorService the ScheduledExecutorService to use * @return itself */ - public T requestExecutorService(final ExecutorService executorService) { + public T requestExecutorService(final ScheduledExecutorService executorService) { this.requestExecutorService = executorService; return self(); } /** - * Adds the HttpClientAdapter to the AbstractEmitter + * Adds a custom HttpClientAdapter to the AbstractEmitter (default is OkHttpClientAdapter). * * @param httpClientAdapter the adapter to use * @return itself @@ -69,19 +70,7 @@ public T httpClientAdapter(final HttpClientAdapter httpClientAdapter) { } /** - * An optional Request Callback for adding the ability to handle failure cases - * for sending. - * - * @param requestCallback the emitter request callback - * @return itself - */ - public T requestCallback(final RequestCallback requestCallback) { - this.requestCallback = requestCallback; - return self(); - } - - /** - * Sets the Thread Count for the ExecutorService + * Sets the Thread Count for the ScheduledExecutorService (default is 50). * * @param threadCount the size of the thread pool * @return itself @@ -92,8 +81,8 @@ public T threadCount(final int threadCount) { } /** - * Sets the emitter url for when a httpClientAdapter is not specified - * Will be used to create the default OkHttpClientAdapter. + * Sets the emitter url for when a httpClientAdapter is not specified. + * It will be used to create the default OkHttpClientAdapter. * * @param collectorUrl the url for the default httpClientAdapter * @return itself @@ -132,8 +121,6 @@ protected AbstractEmitter(final Builder builder) { .build(); } - this.requestCallback = builder.requestCallback; - if (builder.requestExecutorService != null) { this.executor = builder.requestExecutorService; } else { @@ -142,52 +129,43 @@ protected AbstractEmitter(final Builder builder) { } /** - * Adds an event to the buffer + * Adds a payload to the buffer * - * @param event an event + * @param payload an payload */ @Override - public abstract void emit(TrackerEvent event); + public abstract boolean add(TrackerPayload payload); /** - * Customize the emitter buffer size to any valid integer greater than zero. + * Customize the emitter batch size to any valid integer greater than zero. * Has no effect on SimpleEmitter * - * @param bufferSize number of events to collect before sending + * @param batchSize number of events to collect before sending */ @Override - public abstract void setBufferSize(final int bufferSize); + public abstract void setBatchSize(final int batchSize); /** - * Removes all events from the buffer and sends them + * Removes all payloads from the buffer and sends them */ @Override public abstract void flushBuffer(); /** - * Gets the Emitter Buffer Size - Will always be 1 for SimpleEmitter + * Gets the Emitter Batch Size - Will always be 1 for SimpleEmitter * - * @return the buffer size + * @return the batch size */ @Override - public abstract int getBufferSize(); + public abstract int getBatchSize(); /** - * Returns List of Events that are in the buffer. + * Returns List of Payloads that are in the buffer. * * @return the buffered events */ @Override - public abstract List getBuffer(); - - /** - * Sends a runnable to the executor service. - * - * @param runnable the runnable to be queued - */ - protected void execute(final Runnable runnable) { - this.executor.execute(runnable); - } + public abstract List getBuffer(); /** * Checks whether the response code was a success or not. diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitter.java index 94a65927..62e6380d 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -16,55 +16,93 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import com.google.common.base.Preconditions; import com.snowplowanalytics.snowplow.tracker.constants.Constants; import com.snowplowanalytics.snowplow.tracker.constants.Parameter; import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerEvent; import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * An emitter that emit a batch of events in a single call - * It uses the post method of under-laying http adapter + * An emitter that emits a batch of events in a single HTTP request. + * It uses the POST method of the underlying HTTP adapter. + * + * When a new event (TrackerPayload) is received and added to the buffer, the BatchEmitter checks the + * number of buffered events. If it is equal to or greater than the `batchSize`, an attempt is made to send + * a batch of events as one request. Events are sent asynchronously. + * + * If the request is unsuccessful, the events are returned to the buffer. A delay is introduced for all + * event sending attempts. This increases exponentially until a request succeeds, when it is reset to 0. + * Retry will continue indefinitely. + * + * If the buffer becomes full due to network problems, newer events will be lost. */ public class BatchEmitter extends AbstractEmitter implements Closeable { private static final Logger LOGGER = LoggerFactory.getLogger(BatchEmitter.class); - private static final AtomicInteger BUFFER_CONSUMER_THREAD_NUMBER = new AtomicInteger(1); - private static final String BUFFER_CONSUMER_THREAD_NAME_PREFIX = "snowplow-emitter-BufferConsumer-thread-"; - - private final Thread bufferConsumer; private boolean isClosing = false; + + private int batchSize; + private final EventStore eventStore; + private final AtomicLong retryDelay; + private final List fatalResponseCodes; - private int bufferSize = 1; - - // Queue for immediate buffering of events - private final BlockingQueue eventBuffer = new LinkedBlockingQueue<>(); + public static abstract class Builder> extends AbstractEmitter.Builder { - // Queue for storing events until bufferSize is reached - private final BlockingQueue eventsToSend = new LinkedBlockingQueue<>(); + private int batchSize = 50; // Optional + private int bufferCapacity = Integer.MAX_VALUE; + private EventStore eventStore; + private List fatalResponseCodes; - private final long closeTimeout = 5; + /** + * The default batch size is 50. + * + * @param batchSize The count of events to send in one HTTP request + * @return itself + */ + public T batchSize(final int batchSize) { + this.batchSize = batchSize; + return self(); + } - public static abstract class Builder> extends AbstractEmitter.Builder { + /** + * The default EventStore is InMemoryEventStore. + * + * @param eventStore The EventStore to use + * @return itself + */ + public T eventStore(final EventStore eventStore) { + this.eventStore = eventStore; + return self(); + } - private int bufferSize = 50; // Optional + /** + * The default buffer capacity is Integer.MAX_VALUE. Your application would likely run out + * of memory before buffering this many events. When the buffer is full, new events are lost. + * + * @param bufferCapacity The maximum capacity of the default InMemoryEventStore event buffer + * @return itself + */ + public T bufferCapacity(final int bufferCapacity) { + this.bufferCapacity = bufferCapacity; + return self(); + } /** - * @param bufferSize The count of events to buffer before sending + * Provide a denylist of HTTP response codes. Retry will not be attempted if one of these codes + * is received. The events in the request will be dropped, but the Emitter will continue trying + * to send as normal. + * + * @param fatalResponseCodes Event sending will not be retried on these codes * @return itself */ - public T bufferSize(final int bufferSize) { - this.bufferSize = bufferSize; + public T fatalResponseCodes(final List fatalResponseCodes) { + this.fatalResponseCodes = fatalResponseCodes; return self(); } @@ -88,149 +126,137 @@ protected BatchEmitter(final Builder builder) { super(builder); // Precondition checks - Preconditions.checkArgument(builder.bufferSize > 0, "bufferSize must be greater than 0"); + Preconditions.checkArgument(builder.batchSize > 0, "batchSize must be greater than 0"); + batchSize = builder.batchSize; - this.bufferSize = builder.bufferSize; + if (builder.eventStore == null) { + eventStore = new InMemoryEventStore(builder.bufferCapacity); + } else { + eventStore = builder.eventStore; + } + retryDelay = new AtomicLong(0L); - bufferConsumer = new Thread( - getBufferConsumerRunnable(), - BUFFER_CONSUMER_THREAD_NAME_PREFIX + BUFFER_CONSUMER_THREAD_NUMBER.getAndIncrement() - ); - bufferConsumer.start(); + if (builder.fatalResponseCodes != null) { + fatalResponseCodes = builder.fatalResponseCodes; + } else { + fatalResponseCodes = new ArrayList<>(); + } } /** - * Adds a TrackerEvent to the concurrent queue buffer + * Adds a TrackerPayload to the EventStore buffer. + * If the buffer is full, the payload will be lost. + * + *

+ * Implementation note: As a side effect it triggers an Emitter thread to emit a batch of events. * - * @param event an event + * @param payload a TrackerPayload + * @return whether the payload has been successfully added to the buffer. */ @Override - public void emit(final TrackerEvent event) { - boolean result = eventBuffer.offer(event); // Add to buffer and quickly return back to application + public boolean add(final TrackerPayload payload) { + boolean result = eventStore.addEvent(payload); + + if (!isClosing) { + if (eventStore.size() >= batchSize) { + executor.schedule(getPostRequestRunnable(batchSize), retryDelay.get(), TimeUnit.MILLISECONDS); + } + } if (!result) { - LOGGER.error("Unable to add event to emitter, emitter buffer is full"); + LOGGER.error("Unable to add payload to emitter, emitter buffer is full"); } + + return result; } - /* - * Forces the events currently in the buffer to be sent + /** + * Forces all the payloads currently in the buffer to be sent immediately, as a single request. */ @Override public void flushBuffer() { - // Drain immediate event buffer - while (true) { - TrackerEvent event = eventBuffer.poll(); - if (event == null) { - break; - } else { - eventsToSend.offer(event); - } - } - - drainBufferAndSend(); + executor.schedule(getPostRequestRunnable(eventStore.size()), 0, TimeUnit.MILLISECONDS); } /** - * Returns List of Events that are in the buffer. + * Returns a List of Payloads that are in the buffer. * * @return the buffered events */ @Override - public List getBuffer() { - return eventsToSend.stream().collect(Collectors.toList()); + public List getBuffer() { + return eventStore.getAllEvents(); } /** - * Customize the emitter buffer size to any valid integer greater than zero. + * Customize the emitter batch size to any valid integer greater than zero. * - * @param bufferSize number of events to collect before sending + * @param batchSize number of events to send in one request */ @Override - public void setBufferSize(final int bufferSize) { - Preconditions.checkArgument(bufferSize > 0, "bufferSize must be greater than 0"); - this.bufferSize = bufferSize; + public void setBatchSize(final int batchSize) { + Preconditions.checkArgument(batchSize > 0, "batchSize must be greater than 0"); + this.batchSize = batchSize; } /** - * Gets the Emitter Buffer Size + * Gets the Emitter batchSize * - * @return the buffer size + * @return the batch size */ @Override - public int getBufferSize() { - return this.bufferSize; + public int getBatchSize() { + return batchSize; } - /** - * Returns a Consumer for the concurrent queue buffer - * Consumes events onto another queue to be sent when bufferSize is reached - * - * @return the new Runnable object - */ - private Runnable getBufferConsumerRunnable() { - return new Runnable() { - @Override - public void run() { - while (true) { - try { - eventsToSend.put(eventBuffer.take()); - if (eventsToSend.size() >= bufferSize) { - drainBufferAndSend(); - } - } catch (InterruptedException ex) { - if (isClosing) { - return; - } - } - } - } - }; - } - - private void drainBufferAndSend() { - List events = new ArrayList<>(); - eventsToSend.drainTo(events); - execute(getRequestRunnable(events)); + long getRetryDelay() { + return retryDelay.get(); } /** * Returns a Runnable POST Request operation * - * @param buffer the event buffer to be sent + * @param numberOfEvents the number of events to be sent in the request * @return the new Runnable object */ - private Runnable getRequestRunnable(final List buffer) { - return new Runnable() { - @Override - public void run() { - if (buffer.size() == 0) { + private Runnable getPostRequestRunnable(int numberOfEvents) { + return () -> { + BatchPayload batchedEvents = null; + try { + batchedEvents = eventStore.getEventsBatch(numberOfEvents); + + if (batchedEvents == null || batchedEvents.size() == 0) { return; } - final SelfDescribingJson post = getFinalPost(buffer); + List eventsInRequest = batchedEvents.getPayloads(); + final SelfDescribingJson post = getFinalPost(eventsInRequest); final int code = httpClientAdapter.post(post); // Process results - int success = 0; - int failure = 0; - if (!isSuccessfulSend(code)) { - LOGGER.error("BatchEmitter failed to send {} events: code: {}", buffer.size(), code); - failure += buffer.size(); + if (isSuccessfulSend(code)) { + LOGGER.debug("BatchEmitter successfully sent {} events: code: {}", eventsInRequest.size(), code); + retryDelay.set(0L); + eventStore.cleanupAfterSendingAttempt(false, batchedEvents.getBatchId()); + + } else if (fatalResponseCodes.contains(code)) { + LOGGER.debug("BatchEmitter failed to send {} events. No retry for code {}: events dropped", eventsInRequest.size(), code); + eventStore.cleanupAfterSendingAttempt(false, batchedEvents.getBatchId()); + } else { - LOGGER.debug("BatchEmitter successfully sent {} events: code: {}", buffer.size(), code); - success += buffer.size(); - } + LOGGER.error("BatchEmitter failed to send {} events: code: {}", eventsInRequest.size(), code); + eventStore.cleanupAfterSendingAttempt(true, batchedEvents.getBatchId()); - // Send the callback if available - if (requestCallback != null) { - if (failure != 0) { - requestCallback.onFailure(success, - buffer.stream().map(te -> te.getEvent()).collect(Collectors.toList())); - } else { - requestCallback.onSuccess(success); + // exponentially increase retry backoff time after the first failure + if (!retryDelay.compareAndSet(0, 50L)) { + retryDelay.updateAndGet(currentDelay -> currentDelay * 2); } } + } catch (Exception e) { + LOGGER.error("BatchEmitter event sending error: {}", e.getMessage()); + if (batchedEvents != null) { + eventStore.cleanupAfterSendingAttempt(true, batchedEvents.getBatchId()); + } } }; } @@ -238,32 +264,33 @@ public void run() { /** * Constructs the SelfDescribingJson to be sent to the endpoint * - * @param buffer the event buffer + * @param events the event buffer * @return the constructed POST payload */ - private SelfDescribingJson getFinalPost(final List buffer) { + private SelfDescribingJson getFinalPost(final List events) { final List> toSendPayloads = new ArrayList<>(); final String sentTimestamp = Long.toString(System.currentTimeMillis()); - for (TrackerEvent event : buffer) { - List payloads = event.getTrackerPayloads(); - for (TrackerPayload payload : payloads) { - payload.add(Parameter.DEVICE_SENT_TIMESTAMP, sentTimestamp); - toSendPayloads.add(payload.getMap()); - } + for (TrackerPayload payload : events) { + payload.add(Parameter.DEVICE_SENT_TIMESTAMP, sentTimestamp); + toSendPayloads.add(payload.getMap()); } return new SelfDescribingJson(Constants.SCHEMA_PAYLOAD_DATA, toSendPayloads); } /** - * On close attempt to send all remaining events. + * Attempt to send all remaining events, then shut down the ExecutorService. + * + *

+ * Implementation note: Be aware that calling `close()` + * has a side-effect of shutting down the Emitter ScheduledExecutorService. */ @Override public void close() { + final long closeTimeout = 5; isClosing = true; - bufferConsumer.interrupt(); // Kill buffer consumer flushBuffer(); // Attempt to send all remaining events //Shutdown executor threadpool @@ -273,7 +300,7 @@ public void close() { if (!executor.awaitTermination(closeTimeout, TimeUnit.SECONDS)) { executor.shutdownNow(); if (!executor.awaitTermination(closeTimeout, TimeUnit.SECONDS)) - LOGGER.warn("Executor did not terminate"); + LOGGER.warn("Emitter executor did not terminate"); } } catch (final InterruptedException ie) { executor.shutdownNow(); diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchPayload.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchPayload.java new file mode 100644 index 00000000..cf39dd2d --- /dev/null +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchPayload.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.tracker.emitter; + +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; + +import java.util.List; + +/** + * A wrapper for a number of TrackerPayloads. + */ +public class BatchPayload { + + private final Long batchId; + private final List payloads; + + public BatchPayload(Long batchId, List payloads) { + this.batchId = batchId; + this.payloads = payloads; + } + + public Long getBatchId() { + return batchId; + } + + public List getPayloads() { + return payloads; + } + + public int size() { + return payloads.size(); + } +} diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/Emitter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/Emitter.java index c51a456d..b2b21a08 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/Emitter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/Emitter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -14,7 +14,7 @@ import java.util.List; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerEvent; +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; /** * Emitter interface. @@ -22,43 +22,41 @@ public interface Emitter { /** - * Adds an event to the buffer and checks whether + * Adds a payload to the buffer and checks whether * we have reached the buffer limit yet. * - * @param event an event to be emitted + * @param payload a payload to be emitted + * @return if the payload was added to the buffer */ - void emit(TrackerEvent event); + boolean add(TrackerPayload payload); /** - * Customize the emitter buffer size to any valid integer + * Customize the emitter batch size to any valid integer * greater than zero. - * - Will only effect the BatchEmitter + * Will only affect the BatchEmitter * - * @param bufferSize number of events to collect before + * @param batchSize number of events to collect before * sending */ - void setBufferSize(int bufferSize); + void setBatchSize(int batchSize); /** - * When the buffer limit is reached sending of the buffer is - * initiated. - * - * This can be used to manually start sending. + * This can be used to manually send all buffered events. */ void flushBuffer(); /** - * Gets the Emitter Buffer Size - * - Will always be 1 for SimpleEmitter + * Gets the Emitter Batch Size + * Will always be 1 for SimpleEmitter. Note that SimpleEmitter has been deprecated. * - * @return the buffer size + * @return the batch size */ - int getBufferSize(); + int getBatchSize(); /** - * Returns the List of Events that are in the buffer. + * Returns the List of Payloads that are in the buffer. * * @return the buffer events */ - List getBuffer(); + List getBuffer(); } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/EventStore.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/EventStore.java new file mode 100644 index 00000000..3fa80d13 --- /dev/null +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/EventStore.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.tracker.emitter; + +import java.util.List; + +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; + +/** + * EventStore interface. For buffering events in the Emitter. + */ +public interface EventStore { + + /** + * Add TrackerPayload to buffer. + * + * @param trackerPayload the payload to add + * @return success or not + */ + boolean addEvent(TrackerPayload trackerPayload); + + /** + * Remove some TrackerPayloads from the buffer. + * + * @param numberToGet how many payloads to get + * @return a BatchPayload wrapper + */ + BatchPayload getEventsBatch(int numberToGet); + + /** + * Get a copy of all the TrackerPayloads in the buffer. + * + * @return List of all the stored events + */ + List getAllEvents(); + + /** + * Finish processing events after a request has been made. + * + * @param needRetry if another attempt should be made to send the events + * @param batchId the ID of the batch of events + */ + void cleanupAfterSendingAttempt(boolean needRetry, long batchId); + + /** + * Get the current size of the buffer. + * + * @return number of events currently in the buffer + */ + int size(); +} diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/InMemoryEventStore.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/InMemoryEventStore.java new file mode 100644 index 00000000..a698a16b --- /dev/null +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/InMemoryEventStore.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.tracker.emitter; + +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Buffers events (as TrackerPayloads) in memory for sending via the BatchEmitter. + * + * The TrackerPayloads are stored in a queue. When the BatchEmitter calls {@link #getEventsBatch(int)}, + * the chosen number of TrackerPayloads are removed from the queue. The batch is added to a map of payloads + * that are currently being sent, and wrapped as a BatchPayload. This BatchPayload wrapper is returned to the + * Emitter. + * + * If the POST request is successful, the payloads are deleted from the map. + * If not, they are removed from the map and reinserted into the queue to be sent again. + */ +public class InMemoryEventStore implements EventStore { + private static final Logger LOGGER = LoggerFactory.getLogger(InMemoryEventStore.class); + private final AtomicLong batchId = new AtomicLong(1); + + private final LinkedBlockingDeque eventBuffer; + private final ConcurrentHashMap> eventsBeingSent = new ConcurrentHashMap<>(); + + /** + * Make a new InMemoryEventStore with default queue capacity (Integer.MAX_VALUE). + */ + public InMemoryEventStore() { + eventBuffer = new LinkedBlockingDeque<>(); + } + + /** + * Make a new InMemoryEventStore with user-set queue capacity. + * + * @param bufferCapacity the maximum number of events to buffer at once + */ + public InMemoryEventStore(int bufferCapacity) { + eventBuffer = new LinkedBlockingDeque<>(bufferCapacity); + } + + /** + * Add TrackerPayload to buffer. Returns false if the buffer was full. + * Note that the event is lost in this case. + * + * @param trackerPayload the payload to add + * @return success or not + */ + @Override + public boolean addEvent(TrackerPayload trackerPayload) { + return eventBuffer.offer(trackerPayload); + } + + /** + * Remove some TrackerPayloads from the buffer. They are wrapped as a BatchPayload to return, + * and also stored in a separate collection inside InMemoryEventStore until the result of their POST request is known. + * + * @param numberToGet how many payloads to get + * @return a BatchPayload wrapper, or null + */ + @Override + public BatchPayload getEventsBatch(int numberToGet) { + List eventsToSend = new ArrayList<>(); + + synchronized (eventBuffer) { + if (eventBuffer.size() < numberToGet) { + return null; + } + eventBuffer.drainTo(eventsToSend, numberToGet); + } + + // The batch of events is wrapped as a BatchPayload + // They're also added to the "pending" event buffer, the eventsBeingSent HashMap + BatchPayload batchedEvents = new BatchPayload(batchId.getAndIncrement(), eventsToSend); + eventsBeingSent.put(batchedEvents.getBatchId(), batchedEvents.getPayloads()); + return batchedEvents; + } + + /** + * Finish processing events after a request has been made. If the request was successful, + * the events are deleted from the InMemoryEventStore. If not, they are reinserted at the beginning + * of the buffer queue for another attempt. + * + * @param needRetry if true, move events back to the buffer instead of deleting + * @param batchId the ID of the batch of events + */ + @Override + public void cleanupAfterSendingAttempt(boolean needRetry, long batchId) { + // Events that successfully sent are deleted from the pending buffer + List events = eventsBeingSent.remove(batchId); + + // Events that didn't send are inserted at the head of the eventBuffer + // for immediate resending. + if (needRetry) { + while (events.size() > 0) { + TrackerPayload payloadToReinsert = events.remove(0); + boolean result = eventBuffer.offerFirst(payloadToReinsert); + if (!result) { + LOGGER.error("Event buffer is full. Dropping newer payload to reinsert older payload"); + eventBuffer.removeLast(); + eventBuffer.offerFirst(payloadToReinsert); + } + } + } + } + + /** + * Get a copy of all the TrackerPayloads in the buffer. This does not include any events + * currently being sent by the BatchEmitter. + * + * @return List of all the stored events + */ + @Override + public List getAllEvents() { + TrackerPayload[] events = eventBuffer.toArray(new TrackerPayload[0]); + return Arrays.asList(events); + } + + /** + * Get the current size of the buffer. This does not include any events + * currently being sent by the BatchEmitter. + * + * @return number of events currently in the buffer + */ + @Override + public int size() { + return eventBuffer.size(); + } +} diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/RequestCallback.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/RequestCallback.java deleted file mode 100644 index 4df7c8bb..00000000 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/RequestCallback.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.snowplow.tracker.emitter; - -import java.util.List; - -import com.snowplowanalytics.snowplow.tracker.events.Event; - -/** - * Provides a callback interface for reporting counts of successfully sent - * events and returning any failed events to be handled by the developer. - */ -public interface RequestCallback { - - /** - * If all events are sent successfully then the count - * of sent events are returned. - * - * @param successCount the successful count - */ - void onSuccess(int successCount); - - /** - * If all/some events failed then the count of successful - * events is returned along with all the failed Events. - * - * @param successCount the successful count - * @param failedEvents the list of failed events - */ - void onFailure(int successCount, List failedEvents); -} diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/SimpleEmitter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/SimpleEmitter.java index 8b7a46df..8f81dce0 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/SimpleEmitter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/emitter/SimpleEmitter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -18,15 +18,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerEvent; import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; import com.snowplowanalytics.snowplow.tracker.constants.Parameter; -import com.snowplowanalytics.snowplow.tracker.events.Event; /** - * An emitter which sends events as soon as they are received via - * GET requests. + * An emitter which sends events one-by-one as soon as they are received, via + * GET requests. The events are sent asynchronously. + * @deprecated Use the BatchEmitter, or create your own Emitter using the provided interface. */ +@Deprecated public class SimpleEmitter extends AbstractEmitter { private static final Logger LOGGER = LoggerFactory.getLogger(SimpleEmitter.class); @@ -53,17 +53,22 @@ protected SimpleEmitter(final Builder builder) { } /** - * Adds an event to the buffer and instantly sends it + * Adds an event and instantly tries to send it. * - * @param event an event + * @param payload a payload + * @return true */ @Override - public void emit(final TrackerEvent event) { - execute(getRequestRunnable(event)); + public boolean add(TrackerPayload payload) { + executor.execute(getGetRequestRunnable(payload)); + + // This result doesn't mean anything + // The return type is for BatchEmitter's benefit + return true; } /** - * Sends buffered events, but SimpleEmitter does not buffer events + * Sends all the buffered events, but SimpleEmitter does not buffer events. * So has no effect */ @Override @@ -74,42 +79,23 @@ public void flushBuffer() { /** * Returns a Runnable GET Request operation * - * @param event the event to be sent + * @param payload the event to be sent * @return the new Callable object */ - private Runnable getRequestRunnable(final TrackerEvent event) { + private Runnable getGetRequestRunnable(final TrackerPayload payload) { return new Runnable() { @Override public void run() { - int success = 0; - int failure = 0; - - List payloads = event.getTrackerPayloads(); - - for (TrackerPayload payload : payloads) { - payload.add(Parameter.DEVICE_SENT_TIMESTAMP, Long.toString(System.currentTimeMillis())); - final int code = httpClientAdapter.get(payload); - - // Process results - if (!isSuccessfulSend(code)) { - LOGGER.error("SimpleEmitter failed to send {} events: code: {}", 1, code); - failure += 1; - } else { - LOGGER.debug("SimpleEmitter successfully sent {} events: code: {}", 1, code); - success += 1; - } + payload.add(Parameter.DEVICE_SENT_TIMESTAMP, Long.toString(System.currentTimeMillis())); + final int code = httpClientAdapter.get(payload); + + // Process results + if (!isSuccessfulSend(code)) { + LOGGER.error("SimpleEmitter failed to send {} events: code: {}", 1, code); + } else { + LOGGER.debug("SimpleEmitter successfully sent {} events: code: {}", 1, code); } - // Send the callback if available - if (requestCallback != null) { - if (failure != 0) { - final List buffer = new ArrayList<>(); - buffer.add(event.getEvent()); - requestCallback.onFailure(success, buffer); - } else { - requestCallback.onSuccess(success); - } - } } }; } @@ -121,30 +107,30 @@ public void run() { * @return the empty buffer */ @Override - public List getBuffer() { + public List getBuffer() { return new ArrayList<>(); } /** - * Customize the emitter buffer size to any valid integer greater than zero. + * Customize the emitter batch size to any valid integer greater than zero. * Has no effect on SimpleEmitter * - * @param bufferSize number of events to collect before sending + * @param batchSize number of events to collect before sending */ @Override - public void setBufferSize(final int bufferSize) { - if (bufferSize != 1) { - LOGGER.debug("Noop. SimpleEmitter buffer size must always be 1."); + public void setBatchSize(final int batchSize) { + if (batchSize != 1) { + LOGGER.debug("Noop. SimpleEmitter batch size must always be 1."); } } /** - * Gets the Emitter Buffer Size - Will always be 1 for SimpleEmitter + * Gets the Emitter batch Size - Will always be 1 for SimpleEmitter * - * @return the buffer size + * @return the batch size */ @Override - public int getBufferSize() { + public int getBatchSize() { return 1; } } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/AbstractEvent.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/AbstractEvent.java index 31161c0e..d77e2a28 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/AbstractEvent.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/AbstractEvent.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -22,48 +22,42 @@ // This library import com.snowplowanalytics.snowplow.tracker.Subject; -import com.snowplowanalytics.snowplow.tracker.Utils; import com.snowplowanalytics.snowplow.tracker.constants.Parameter; import com.snowplowanalytics.snowplow.tracker.payload.Payload; import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; /** - * Base AbstractEvent class which contains common - * elements to all events: - * - Custom Context: list of custom contexts or null - * - Timestamp: user defined event timestamp or 0 - * - Event Id: a unique id for the event - * - Subject: a unique Subject for the event + * Base AbstractEvent class which contains + * elements that can be set in all events. These are context, trueTimestamp, and Subject. + * + * Context is a list of custom SelfDescribingJson entities. + * TrueTimestamp is a user-defined timestamp. + * Subject is an event-specific Subject. Its fields will override those of the + * Tracker-associated Subject, if present. */ public abstract class AbstractEvent implements Event { protected final List context; - protected long deviceCreatedTimestamp; - /** - * The true timestamp may be null if none is set. + * The trueTimestamp may be null if none is set. */ protected Long trueTimestamp; - - protected final String eventId; protected final Subject subject; public static abstract class Builder> { private List context = new LinkedList<>(); - private long deviceCreatedTimestamp = System.currentTimeMillis(); protected Long trueTimestamp = null; - private String eventId = Utils.getEventId(); private Subject subject = null; protected abstract T self(); /** - * Adds a list of custom contexts. + * Adds a list of custom context entities. * - * @param context the list of contexts + * @param context the list of entities * @return itself */ public T customContext(List context) { @@ -71,31 +65,6 @@ public T customContext(List context) { return self(); } - /** - * A custom event timestamp. - * - * @param timestamp the event timestamp as - * unix epoch - * @return itself - * Use {@link #trueTimestamp} or {@link #deviceCreatedTimestamp} - */ - @Deprecated - public T timestamp(long timestamp) { - return deviceCreatedTimestamp(timestamp); - } - - /** - * Adjust the device-created timestamp. This is usually not what you want, check {@link #trueTimestamp}. - * - * @param timestamp the event timestamp as - * unix epoch - * @return itself - */ - public T deviceCreatedTimestamp(long timestamp) { - this.deviceCreatedTimestamp = timestamp; - return self(); - } - /** * The true timestamp of that event (as determined by the user). * @@ -109,18 +78,8 @@ public T trueTimestamp(Long timestamp) { } /** - * A custom eventId for the event. - * - * @param eventId the eventId - * @return itself - */ - public T eventId(String eventId) { - this.eventId = eventId; - return self(); - } - - /** - * A custom subject for the event. + * A custom Subject for the event. Its fields will override those of the + * Tracker-associated Subject, if present. * * @param subject the eventId * @return itself @@ -146,13 +105,9 @@ protected AbstractEvent(Builder builder) { // Precondition checks Preconditions.checkNotNull(builder.context); - Preconditions.checkNotNull(builder.eventId); - Preconditions.checkArgument(!builder.eventId.isEmpty(), "eventId cannot be empty"); this.context = builder.context; - this.deviceCreatedTimestamp = builder.deviceCreatedTimestamp; this.trueTimestamp = builder.trueTimestamp; - this.eventId = builder.eventId; this.subject = builder.subject; } @@ -164,23 +119,6 @@ public List getContext() { return new ArrayList<>(this.context); } - /** - * @return the event's timestamp - * @deprecated Use {@link #getTrueTimestamp()} or {@link #getDeviceCreatedTimestamp()} - */ - @Override - public long getTimestamp() { - return this.deviceCreatedTimestamp; - } - - /** - * @return the event's device created timestamp. - */ - @Override - public long getDeviceCreatedTimestamp() { - return deviceCreatedTimestamp; - } - /** * @return the event's true timestamp. */ @@ -189,14 +127,6 @@ public Long getTrueTimestamp() { return trueTimestamp; } - /** - * @return the event id - */ - @Override - public String getEventId() { - return this.eventId; - } - /** * @return the event subject */ @@ -214,15 +144,13 @@ public Subject getSubject() { /** * Adds the default parameters to a TrackerPayload object. * - * @param payload the payload to add too. + * @param payload the payload to add to. * @return the TrackerPayload with appended values. */ - protected TrackerPayload putDefaultParams(TrackerPayload payload) { - payload.add(Parameter.EID, getEventId()); - if (getTrueTimestamp()!=null) { + protected TrackerPayload putTrueTimestamp(TrackerPayload payload) { + if (getTrueTimestamp() != null) { payload.add(Parameter.TRUE_TIMESTAMP, Long.toString(getTrueTimestamp())); } - payload.add(Parameter.DEVICE_CREATED_TIMESTAMP, Long.toString(getDeviceCreatedTimestamp())); return payload; } } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransaction.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransaction.java index d7f31752..1a14af15 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransaction.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransaction.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -25,6 +25,24 @@ import com.snowplowanalytics.snowplow.tracker.constants.Constants; import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; +/** + * Constructs an EcommerceTransaction event object. + *

+ * Implementation note: EcommerceTransaction/EcommerceTransactionItem uses a legacy design. + * We aim to deprecate it eventually. We advise using Unstructured events instead, and attaching the items + * as entities. + * + * The specific items purchased in the transaction must be added as EcommerceTransactionItem objects. + * This event type is different from the others in that it will generate more than one tracked event. + * There will be one "transaction" ("tr") event, and one "transaction item" ("ti") event for every + * EcommerceTransactionItem included in the EcommerceTransaction. + * + * To link the "transaction" and "transaction item" events, we recommend using the same orderId for the + * EcommerceTransaction and all attached EcommerceTransactionItems. + * + * To use the Currency Conversion pipeline enrichment, the currency string must be + * a valid Open Exchange Rates value. + */ public class EcommerceTransaction extends AbstractEvent { private final String orderId; @@ -52,6 +70,8 @@ public static abstract class Builder> extends AbstractEvent private List items; /** + * Required. + * * @param orderId ID of the eCommerce transaction * @return itself */ @@ -61,6 +81,8 @@ public T orderId(String orderId) { } /** + * Required. + * * @param totalValue Total transaction value * @return itself */ @@ -70,6 +92,8 @@ public T totalValue(Double totalValue) { } /** + * Optional. + * * @param affiliation Transaction affiliation * @return itself */ @@ -79,6 +103,8 @@ public T affiliation(String affiliation) { } /** + * Optional. + * * @param taxValue Transaction tax value * @return itself */ @@ -88,6 +114,8 @@ public T taxValue(Double taxValue) { } /** + * Optional. + * * @param shipping Delivery cost charged * @return itself */ @@ -97,6 +125,8 @@ public T shipping(Double shipping) { } /** + * Optional. + * * @param city Delivery address city * @return itself */ @@ -106,6 +136,8 @@ public T city(String city) { } /** + * Optional. + * * @param state Delivery address state * @return itself */ @@ -115,6 +147,8 @@ public T state(String state) { } /** + * Optional. + * * @param country Delivery address country * @return itself */ @@ -124,6 +158,8 @@ public T country(String country) { } /** + * Optional. + * * @param currency The currency the price is expressed in * @return itself */ @@ -133,6 +169,9 @@ public T currency(String currency) { } /** + * Provide a list of EcommerceTransactionItems. + * An empty list is valid, but probably not very useful. + * * @param items The items in the transaction * @return itself */ @@ -142,6 +181,9 @@ public T items(List items) { } /** + * Provide EcommerceTransactionItems directly, without explicitly adding them + * to a list beforehand. + * * @param itemArgs The items as a varargs argument * @return itself */ @@ -190,8 +232,7 @@ protected EcommerceTransaction(Builder builder) { } /** - * Returns a TrackerPayload which can be stored into - * the local database. + * Returns a TrackerPayload which can be passed to an Emitter. * * @return the payload to be sent. */ @@ -209,11 +250,11 @@ public TrackerPayload getPayload() { payload.add(Parameter.TR_STATE, this.state); payload.add(Parameter.TR_COUNTRY, this.country); payload.add(Parameter.TR_CURRENCY, this.currency); - return putDefaultParams(payload); + return putTrueTimestamp(payload); } /** - * The list of Transaction Items passed with the event. + * The list of EcommerceTransactionItems passed with the event. * * @return the items. */ diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransactionItem.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransactionItem.java index 0a928f4c..aa4b4ab5 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransactionItem.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/EcommerceTransactionItem.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -21,6 +21,23 @@ import com.snowplowanalytics.snowplow.tracker.constants.Constants; import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; +/** + * Constructs an EcommerceTransactionItem object. + *

+ * Implementation note: EcommerceTransaction/EcommerceTransactionItem uses a legacy design. + * We aim to deprecate it eventually. We advise using Unstructured events instead, and attaching the items + * as entities. + * + * EcommerceTransactionItems were designed for attaching data about purchased items to a + * EcommerceTransaction event. They can technically be sent as events in their own right, but this is + * not supported. + * + * To link the "transaction" and "transaction item" events, we recommend using the same orderId for the + * EcommerceTransaction and all attached EcommerceTransactionItems. + * + * To use the Currency Conversion pipeline enrichment, the currency string must be + * a valid Open Exchange Rates value. + */ public class EcommerceTransactionItem extends AbstractEvent { private final String itemId; @@ -42,7 +59,9 @@ public static abstract class Builder> extends AbstractEvent private String currency; /** - * @param itemId Item ID + * Required. + * + * @param itemId Item ID - ideally the same as the EcommerceTransaction orderId * @return itself */ public T itemId(String itemId) { @@ -51,6 +70,8 @@ public T itemId(String itemId) { } /** + * Required. + * * @param sku Item SKU * @return itself */ @@ -60,6 +81,8 @@ public T sku(String sku) { } /** + * Required. + * * @param price Item price * @return itself */ @@ -69,6 +92,8 @@ public T price(Double price) { } /** + * Required. + * * @param quantity Item quantity * @return itself */ @@ -78,6 +103,8 @@ public T quantity(Integer quantity) { } /** + * Optional. + * * @param name Item name * @return itself */ @@ -87,6 +114,8 @@ public T name(String name) { } /** + * Optional. + * * @param category Item category * @return itself */ @@ -96,6 +125,8 @@ public T category(String category) { } /** + * Optional. + * * @param currency The currency the price is expressed in * @return itself */ @@ -141,31 +172,7 @@ protected EcommerceTransactionItem(Builder builder) { } /** - * @param timestamp the new timestamp - * Use {@link #setTrueTimestamp(long)} or {@link #setTrueTimestamp(long)} - */ - @Deprecated - public void setTimestamp(long timestamp) { - setDeviceCreatedTimestamp(timestamp); - } - - /** - * @param timestamp the new timestamp - */ - public void setTrueTimestamp(long timestamp) { - this.trueTimestamp = timestamp; - } - - /** - * @param timestamp the new timestamp - */ - public void setDeviceCreatedTimestamp(Long timestamp) { - this.deviceCreatedTimestamp = timestamp; - } - - /** - * Returns a TrackerPayload which can be stored into - * the local database. + * Returns a TrackerPayload which can be passed to an Emitter. * * @return the payload to be sent. */ @@ -179,6 +186,6 @@ public TrackerPayload getPayload() { payload.add(Parameter.TI_ITEM_PRICE, Double.toString(this.price)); payload.add(Parameter.TI_ITEM_QUANTITY, Integer.toString(this.quantity)); payload.add(Parameter.TI_ITEM_CURRENCY, this.currency); - return putDefaultParams(payload); + return putTrueTimestamp(payload); } } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Event.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Event.java index 935279a1..0c0ed380 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Event.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Event.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -24,34 +24,17 @@ public interface Event { /** - * @return the events custom context + * @return the event's custom context */ List getContext(); - /** - * @return the event's timestamp - * Use {@link #getTrueTimestamp()} or {@link #getDeviceCreatedTimestamp()} - */ - @Deprecated - long getTimestamp(); - /** * @return the event's true timestamp */ Long getTrueTimestamp(); /** - * @return the event's device created timestamp - */ - long getDeviceCreatedTimestamp(); - - /** - * @return the event id - */ - String getEventId(); - - /** - * @return the event subject + * @return the event-associated Subject */ Subject getSubject(); diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/PageView.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/PageView.java index 27bed72f..da9becbb 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/PageView.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/PageView.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -22,6 +22,8 @@ /** * Constructs a PageView event object. + * + * When tracked, generates a "pv" or "page_view" event. */ public class PageView extends AbstractEvent { @@ -36,6 +38,8 @@ public static abstract class Builder> extends AbstractEvent private String referrer; /** + * Required. + * * @param pageUrl URL of the viewed page * @return itself */ @@ -45,6 +49,8 @@ public T pageUrl(String pageUrl) { } /** + * Optional. + * * @param pageTitle Title of the viewed page * @return itself */ @@ -54,7 +60,9 @@ public T pageTitle(String pageTitle) { } /** - * @param referrer Referrer of the page + * Optional. + * + * @param referrer Referrer URL of the page * @return itself */ public T referrer(String referrer) { @@ -91,8 +99,7 @@ protected PageView(Builder builder) { } /** - * Returns a TrackerPayload which can be stored into - * the local database. + * Returns a TrackerPayload which can be passed to an Emitter. * * @return the payload to be sent. */ @@ -102,6 +109,6 @@ public TrackerPayload getPayload() { payload.add(Parameter.PAGE_URL, this.pageUrl); payload.add(Parameter.PAGE_TITLE, this.pageTitle); payload.add(Parameter.PAGE_REFR, this.referrer); - return putDefaultParams(payload); + return putTrueTimestamp(payload); } } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/ScreenView.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/ScreenView.java index d84adc57..7fe2691b 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/ScreenView.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/ScreenView.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -17,8 +17,14 @@ import com.snowplowanalytics.snowplow.tracker.constants.Parameter; import com.snowplowanalytics.snowplow.tracker.constants.Constants; import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; +import java.util.LinkedHashMap; + +/** + * Constructs a ScreenView event object. + * + * When tracked, generates an "unstructured" or "ue" event. + */ public class ScreenView extends AbstractEvent { private final String name; @@ -30,7 +36,9 @@ public static abstract class Builder> extends AbstractEvent private String id; /** - * @param name The name of the screen view event + * One of name or id is required. + * + * @param name The (human-readable) name of the screen view * @return itself */ public T name(String name) { @@ -39,6 +47,8 @@ public T name(String name) { } /** + * One of name or id is required. + * * @param id Screen view ID * @return itself */ @@ -74,14 +84,15 @@ protected ScreenView(Builder builder) { } /** - * Return the payload wrapped into a SelfDescribingJson. + * Return the payload wrapped into a SelfDescribingJson. When a ScreenView is tracked, + * the Tracker creates and tracks an Unstructured event from this SelfDescribingJson. * * @return the payload as a SelfDescribingJson. */ public SelfDescribingJson getPayload() { - TrackerPayload payload = new TrackerPayload(); - payload.add(Parameter.SV_ID, this.id); - payload.add(Parameter.SV_NAME, this.name); + LinkedHashMap payload = new LinkedHashMap<>(); + payload.put(Parameter.SV_ID, this.id); + payload.put(Parameter.SV_NAME, this.name); return new SelfDescribingJson(Constants.SCHEMA_SCREEN_VIEW, payload); } } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Structured.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Structured.java index e80da843..95f7fb43 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Structured.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Structured.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -22,6 +22,16 @@ /** * Constructs a Structured event object. + * + * This event type is provided to be roughly equivalent to Google Analytics-style events. + * Note that it is not automatically clear what data should be placed in what field. + * To aid data quality and modeling, agree on business-wide definitions when designing + * your tracking strategy. + * + * We recommend using Unstructured - fully custom - events instead. + * + * When tracked, generates a "struct" or "se" event. + * */ public class Structured extends AbstractEvent { @@ -40,6 +50,8 @@ public static abstract class Builder> extends AbstractEvent private Double value; /** + * Required. + * * @param category Category of the event * @return itself */ @@ -49,7 +61,9 @@ public T category(String category) { } /** - * @param action The event itself + * Required. + * + * @param action Describes what happened in the event * @return itself */ public T action(String action) { @@ -58,7 +72,9 @@ public T action(String action) { } /** - * @param label Refer to the object the action is performed on + * Optional. + * + * @param label Refers to the object the action is performed on * @return itself */ public T label(String label) { @@ -67,6 +83,8 @@ public T label(String label) { } /** + * Optional. + * * @param property Property associated with either the action or the object * @return itself */ @@ -76,6 +94,8 @@ public T property(String property) { } /** + * Optional. + * * @param value A value associated with the user action * @return itself */ @@ -117,8 +137,7 @@ protected Structured(Builder builder) { } /** - * Returns a TrackerPayload which can be stored into - * the local database. + * Returns a TrackerPayload which can be passed to an Emitter. * * @return the payload to be sent. */ @@ -131,6 +150,6 @@ public TrackerPayload getPayload() { payload.add(Parameter.SE_PROPERTY, this.property); payload.add(Parameter.SE_VALUE, this.value != null ? Double.toString(this.value) : null); - return putDefaultParams(payload); + return putTrueTimestamp(payload); } } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Timing.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Timing.java index c005d158..f61370dc 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Timing.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Timing.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -23,6 +23,11 @@ import com.snowplowanalytics.snowplow.tracker.constants.Constants; import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; +/** + * Constructs a Timing event object. + * + * When tracked, generates an "unstructured" or "ue" event. + */ public class Timing extends AbstractEvent { private final String category; @@ -38,6 +43,8 @@ public static abstract class Builder> extends AbstractEvent private String label; /** + * Required. + * * @param category The category of the timed event * @return itself */ @@ -47,6 +54,8 @@ public T category(String category) { } /** + * Required. + * * @param variable Identify the timing being recorded * @return itself */ @@ -56,6 +65,8 @@ public T variable(String variable) { } /** + * Required. + * * @param timing The number of milliseconds in elapsed time to report * @return itself */ @@ -65,6 +76,8 @@ public T timing(Integer timing) { } /** + * Optional. + * * @param label Optional description of this timing * @return itself */ @@ -106,7 +119,8 @@ protected Timing(Builder builder) { } /** - * Return the payload wrapped into a SelfDescribingJson. + * Return the payload wrapped into a SelfDescribingJson. When a Timing event is tracked, + * the Tracker creates and tracks an Unstructured event from this SelfDescribingJson. * * @return the payload as a SelfDescribingJson. */ diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Unstructured.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Unstructured.java index a73b4575..31322215 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Unstructured.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/events/Unstructured.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -23,6 +23,11 @@ /** * Constructs an Unstructured event object. + * + * This is a customisable event type which allows you to track anything describable + * by a JsonSchema. + * + * When tracked, generates an "unstructured" or "ue" event. */ public class Unstructured extends AbstractEvent { @@ -34,9 +39,10 @@ public static abstract class Builder> extends AbstractEvent private SelfDescribingJson eventData; /** - * @param selfDescribingJson The properties of the event. Has two field: - * A "data" field containing the event properties and - * A "schema" field identifying the schema against which the data is validated + * Required. + * + * @param selfDescribingJson The properties of the event. Has two fields: "data", containing the event properties, + * and "schema", identifying the schema against which the data is validated * @return itself */ public T eventData(SelfDescribingJson selfDescribingJson) { @@ -77,8 +83,7 @@ public void setBase64Encode(boolean base64Encode) { } /** - * Returns a TrackerPayload which can be stored into - * the local database. + * Returns a TrackerPayload which can be passed to an Emitter. * * @return the payload to be sent. */ @@ -89,6 +94,6 @@ public TrackerPayload getPayload() { payload.add(Parameter.EVENT, Constants.EVENT_UNSTRUCTURED); payload.addMap(envelope.getMap(), this.base64Encode, Parameter.UNSTRUCTURED_ENCODED, Parameter.UNSTRUCTURED); - return putDefaultParams(payload); + return putTrueTimestamp(payload); } } diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/AbstractHttpClientAdapter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/AbstractHttpClientAdapter.java index 0623966e..451d5aa7 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/AbstractHttpClientAdapter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/AbstractHttpClientAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/ApacheHttpClientAdapter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/ApacheHttpClientAdapter.java index 121e24a4..5620d88a 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/ApacheHttpClientAdapter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/ApacheHttpClientAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapter.java index bbf00bdf..e971d6a7 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/OkHttpClientAdapter.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/OkHttpClientAdapter.java index 3e0a63e6..98dc6026 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/http/OkHttpClientAdapter.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/http/OkHttpClientAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -142,7 +142,7 @@ public int doPost(String url, String payload) { } catch (IOException e) { LOGGER.error("OkHttpClient POST Request failed: {}", e.getMessage()); } - + return returnValue; } -} \ No newline at end of file +} diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/Payload.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/Payload.java index e6febee3..4014fc05 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/Payload.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/Payload.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -21,9 +21,9 @@ public interface Payload { /** - * Add a key-value pair to the payload: - * - Checks that the key is not null or empty - * - Checks that the value is not null or empty + * Add a key-value pair to the payload. + * + * It checks that neither the key nor value is null or empty. * * @param key The parameter key * @param value The parameter value as a String @@ -31,8 +31,8 @@ public interface Payload { void add(String key, String value); /** - * Add all the mappings from the specified map. The effect is the equivalent to that of calling: - * - add(String key, String value) for each key value pair. + * Add all the mappings from the specified map. The effect is the equivalent to that of calling + * {@link #add(String, String)} for each key value pair. * * @param map Key-Value pairs to be stored in this payload */ diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJson.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJson.java index a41285ea..155228dc 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJson.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJson.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -24,9 +24,9 @@ import com.snowplowanalytics.snowplow.tracker.constants.Parameter; /** - * Builds a SelfDescribingJson object which can contain two fields: - * - Schema: the JsonSchema path for this Json - * - Data: the data for this Json + * Builds a SelfDescribingJson object. SelfDescribingJson must contain only two fields, schema and data. + * + * Schema is the JsonSchema path for this Json. Data is the data. */ public class SelfDescribingJson implements Payload { @@ -35,7 +35,7 @@ public class SelfDescribingJson implements Payload { /** * Creates a SelfDescribingJson with only a Schema - * String and an empty data map. + * String and an empty data map. Data can be added later using setData() methods. * * @param schema the schema string */ @@ -47,6 +47,11 @@ public SelfDescribingJson(String schema) { * Creates a SelfDescribingJson with a Schema and a * TrackerPayload object. * + * Note that TrackerPayload objects are initialised with an eventId UUID and + * timestamp (deviceCreatedTimestamp), as they are the basis for sending events. + * Therefore, your SelfDescribingJson data will contain the keys "eid" and "dtm". + * This is unlikely to be what you want. + * * @param schema the schema string * @param data a TrackerPayload object to be embedded as * the data @@ -59,7 +64,7 @@ public SelfDescribingJson(String schema, TrackerPayload data) { /** * Creates a SelfDescribingJson with a Schema and a * SelfDescribingJson object. This can be used to - * nest SDJs inside of each other. + * nest SDJs inside each other. * * @param schema the schema string * @param data a SelfDescribingJson object to be embedded as @@ -96,8 +101,12 @@ public SelfDescribingJson setSchema(String schema) { } /** - * Adds data to the SelfDescribingJson - * - Accepts a TrackerPayload object + * Adds data to the SelfDescribingJson from a TrackerPayload object. + * + * Note that TrackerPayload objects are initialised with an eventId UUID and + * timestamp (deviceCreatedTimestamp), as they are the basis for sending events. + * Therefore, your SelfDescribingJson data will contain the keys "eid" and "dtm". + * This is unlikely to be what you want. * * @param data the data to be added to the SelfDescribingJson * @return this SelfDescribingJson diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerEvent.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerEvent.java deleted file mode 100644 index ced65783..00000000 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerEvent.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. - * - * This program is licensed to you under the Apache License Version 2.0, - * and you may not use this file except in compliance with the Apache License Version 2.0. - * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the Apache License Version 2.0 is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. - */ -package com.snowplowanalytics.snowplow.tracker.payload; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import com.snowplowanalytics.snowplow.tracker.Subject; -import com.snowplowanalytics.snowplow.tracker.constants.Constants; -import com.snowplowanalytics.snowplow.tracker.constants.Parameter; -import com.snowplowanalytics.snowplow.tracker.events.*; - -/** - * A TrackerEvent which allows the TrackerPayload to be filled later. The payload will be - * filled by the Emitter in the Emitter thread, using the getTrackerPayload() method. - */ -public class TrackerEvent { - - private final Event event; - private final TrackerParameters parameters; - private final Subject subject; - - public TrackerEvent(final Event event, final TrackerParameters parameters, final Subject subject) { - this.event = event; - this.parameters = parameters; - this.subject = subject; - } - - /** - * Returns the {@link Event} - * - * @return The {@link Event} - */ - public Event getEvent() { - return this.event; - } - - /** - * Converts a {@link Event} to a list of {@link TrackerPayload} and caches the values. - * Returns a list as some Events contain nested payloads (e.g. {@link EcommerceTransaction}) - * Adds fields to the {@link TrackerPayload} based on the type of the {@link Event}. - * - * @return The populated TrackerPayloads - */ - public List getTrackerPayloads() { - final List payloads = new ArrayList<>(); - final List contexts = event.getContext(); - final Subject subject = event.getSubject(); - - // Figure out what type of event it is - final Class eventClass = event.getClass(); - - if (eventClass.equals(Unstructured.class)) { - - // Need to set the Base64 rule for Unstructured events - final Unstructured unstructured = (Unstructured) event; - unstructured.setBase64Encode(this.parameters.getBase64Encoded()); - TrackerPayload payload = unstructured.getPayload(); - addTrackerParameters(payload); - addContextsAndSubject(contexts, subject, payload); - payloads.add(payload); - } else if (eventClass.equals(Timing.class) || eventClass.equals(ScreenView.class)) { - - // These are wrapper classes for Unstructured events; need to create - // Unstructured events from them and resend. - final Unstructured unstructured = Unstructured.builder() - .eventData((SelfDescribingJson) event.getPayload()) - .customContext(contexts) - .deviceCreatedTimestamp(event.getDeviceCreatedTimestamp()) - .trueTimestamp(event.getTrueTimestamp()) - .eventId(event.getEventId()) - .subject(subject) - .build(); - - unstructured.setBase64Encode(this.parameters.getBase64Encoded()); - TrackerPayload payload = unstructured.getPayload(); - addTrackerParameters(payload); - addContextsAndSubject(contexts, subject, payload); - payloads.add(payload); - } else if (eventClass.equals(EcommerceTransaction.class)) { - - final EcommerceTransaction ecommerceTransaction = (EcommerceTransaction) event; - TrackerPayload payload = ecommerceTransaction.getPayload(); - addTrackerParameters(payload); - addContextsAndSubject(contexts, subject, payload); - payloads.add(payload); - - // Track each item individually - for (final EcommerceTransactionItem item : ecommerceTransaction.getItems()) { - - item.setDeviceCreatedTimestamp(ecommerceTransaction.getDeviceCreatedTimestamp()); - TrackerPayload itemPayload = item.getPayload(); - addTrackerParameters(itemPayload); - addContextsAndSubject(item.getContext(), item.getSubject(), itemPayload); - payloads.add(itemPayload); - } - } else { - - // For all other events, simply get the payload - TrackerPayload payload = (TrackerPayload) event.getPayload(); - addTrackerParameters(payload); - addContextsAndSubject(contexts, subject, payload); - payloads.add(payload); - } - - return payloads; - } - - /** - * Adds the context and subject to the event payload - * - * @param contexts the base event context - can be null or empty - * @param subject the event subject - can be null - * @param payload the payload to add the contexts and subjects to - */ - private void addContextsAndSubject(final List contexts, final Subject subject, TrackerPayload payload) { - // Build the final context and add it to the payload - if (contexts != null && contexts.size() > 0) { - SelfDescribingJson envelope = getFinalContext(contexts); - payload.addMap(envelope.getMap(), this.parameters.getBase64Encoded(), Parameter.CONTEXT_ENCODED, Parameter.CONTEXT); - } - - // Add subject if available - if (subject != null) { - payload.addMap(new HashMap<>(subject.getSubject())); - } else if (this.subject != null) { - payload.addMap(new HashMap<>(this.subject.getSubject())); - } - } - - /** - * Builds the final event context. - * - * @param contexts the base event context - * @return the final event context json with many contexts inside - */ - private SelfDescribingJson getFinalContext(List contexts) { - List> contextMaps = new LinkedList<>(); - for (SelfDescribingJson selfDescribingJson : contexts) { - contextMaps.add(selfDescribingJson.getMap()); - } - return new SelfDescribingJson(Constants.SCHEMA_CONTEXTS, contextMaps); - } - - private void addTrackerParameters(TrackerPayload payload) { - payload.add(Parameter.PLATFORM, this.parameters.getPlatform().toString()); - payload.add(Parameter.APP_ID, this.parameters.getAppId()); - payload.add(Parameter.NAMESPACE, this.parameters.getNamespace()); - payload.add(Parameter.TRACKER_VERSION, this.parameters.getTrackerVersion()); - } -} diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerParameters.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerParameters.java index 90bac316..260b27a3 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerParameters.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerParameters.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -15,9 +15,7 @@ import com.snowplowanalytics.snowplow.tracker.DevicePlatform; /** - * A TrackerEvent which allows the TrackerPayload to be filled later. The - * payload will be filled by the Emitter in the Emitter thread, using the - * getTrackerPayload() method. + * A wrapper for Tracker properties. */ public class TrackerParameters { diff --git a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayload.java b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayload.java index 93bb9303..d3328e11 100644 --- a/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayload.java +++ b/src/main/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayload.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -16,23 +16,49 @@ import java.util.LinkedHashMap; import java.util.Map; +import com.snowplowanalytics.snowplow.tracker.constants.Parameter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.snowplowanalytics.snowplow.tracker.Utils; /** - * Returns a standard Tracker Payload consisting of - * many key - pair values. + * A TrackerPayload stores a map of key - pair values. + * + * When the Emitter attempts to send a TrackerPayload, these pairs are extracted + * and added to the HTTP request (via a SelfDescribingJson). + * The deviceSentTimestamp ("stm") is added at that point. + * + * EventId and deviceCreatedTimestamp are added to the internal map at + * TrackerPayload initialization. */ public class TrackerPayload implements Payload { private static final Logger LOGGER = LoggerFactory.getLogger(TrackerPayload.class); protected final Map payload = new LinkedHashMap<>(); + private final String eventId; + private final Long deviceCreatedTimestamp; + + + public TrackerPayload() { + eventId = Utils.getEventId(); + deviceCreatedTimestamp = System.currentTimeMillis(); + + add(Parameter.EID, eventId); + add(Parameter.DEVICE_CREATED_TIMESTAMP, Long.toString(deviceCreatedTimestamp)); + } + + public String getEventId() { + return eventId; + } + + public Long getDeviceCreatedTimestamp() { + return deviceCreatedTimestamp; + } /** - * Add a key-value pair to the payload: - Checks that the key is not null or - * empty - Checks that the value is not null or empty + * Add a key-value pair to the payload. + * Checks that neither the key nor the value are null or empty. * * @param key The parameter key * @param value The parameter value as a String @@ -53,7 +79,7 @@ public void add(final String key, final String value) { /** * Add all the mappings from the specified map. The effect is the equivalent to - * that of calling: - add(String key, String value) for each key value pair. + * that of calling {@link #add(String, String)} for each key value pair. * * @param map Key-Value pairs to be stored in this payload */ diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/SubjectTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/SubjectTest.java index b2850d2c..9ef9352e 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/SubjectTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/SubjectTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/TrackerTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/TrackerTest.java index 1743c9e3..3ddfd1ae 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/TrackerTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/TrackerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -25,36 +25,27 @@ import com.snowplowanalytics.snowplow.tracker.emitter.Emitter; import com.snowplowanalytics.snowplow.tracker.events.*; import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerEvent; public class TrackerTest { public static final String EXPECTED_CONTEXTS = "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1\",\"data\":[{\"schema\":\"schema\",\"data\":{\"foo\":\"bar\"}}]}"; - public static final String EXPECTED_EVENT_ID = "15e9b149-6029-4f6e-8447-5b9797c9e6be"; public static class MockEmitter implements Emitter { - public ArrayList eventList = new ArrayList<>(); + public ArrayList eventList = new ArrayList<>(); @Override - public void emit(TrackerEvent event) { - eventList.add(event); + public boolean add(TrackerPayload payload) { + eventList.add(payload); + return true; } - @Override - public void setBufferSize(int bufferSize) {} - + public void setBatchSize(int batchSize) {} @Override public void flushBuffer() {} - @Override - public int getBufferSize() { - return 0; - } - + public int getBatchSize() { return 0; } @Override - public List getBuffer() { - return null; - } + public List getBuffer() { return null; } } MockEmitter mockEmitter; @@ -75,7 +66,58 @@ public void setUp() { // --- Event Tests @Test - public void testEcommerceEvent() { + public void testTrackReturnsEventIdIfSuccessful() throws InterruptedException { + // a list to allow for eCommerceTransaction + List result = tracker.track(Unstructured.builder() + .eventData(new SelfDescribingJson( + "iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0", + ImmutableMap.of("foo", "bar") + )) + .build()); + + Thread.sleep(500); + + boolean isValidEventId = true; + try { + UUID.fromString(result.get(0)); + } catch (Exception e) { + isValidEventId = false; + } + + assertTrue(isValidEventId); + } + + @Test + public void testTrackReturnsNullIfEventWasDropped() throws InterruptedException { + class FailingMockEmitter implements Emitter { + @Override + public boolean add(TrackerPayload payload) { return false; } + @Override + public void setBatchSize(int batchSize) {} + @Override + public void flushBuffer() {} + @Override + public int getBatchSize() { return 0; } + @Override + public List getBuffer() { return null; } + } + FailingMockEmitter failingMockEmitter = new FailingMockEmitter(); + tracker = new Tracker.TrackerBuilder(failingMockEmitter, "AF003", "cloudfront").build(); + + List result = tracker.track(Unstructured.builder() + .eventData(new SelfDescribingJson( + "iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0", + ImmutableMap.of("foo", "bar") + )) + .build()); + + Thread.sleep(500); + + assertNull(result.get(0)); + } + + @Test + public void testEcommerceEvent() throws InterruptedException { // Given EcommerceTransactionItem item = EcommerceTransactionItem.builder() .itemId("order_id") @@ -86,9 +128,7 @@ public void testEcommerceEvent() { .category("category") .currency("currency") .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build(); // When @@ -104,25 +144,23 @@ public void testEcommerceEvent() { .currency("currency") .items(item) .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - List results = mockEmitter.eventList.get(0).getTrackerPayloads(); + Thread.sleep(500); + + List results = mockEmitter.eventList; assertEquals(2, results.size()); Map result1 = results.get(0).getMap(); - assertEquals(ImmutableMap.builder() + Map expected1 = ImmutableMap.builder() .put("e", "tr") .put("tr_cu", "currency") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("aid", "cloudfront") .put("tr_sh", "3.0") - .put("dtm", "123456") .put("ttm", "456789") .put("tz", "Etc/UTC") .put("tr_co", "country") @@ -134,19 +172,61 @@ public void testEcommerceEvent() { .put("tr_tt", "1.0") .put("tr_ci", "city") .put("tr_st", "state") - .build(), result1); + .build(); + + assertTrue(result1.entrySet().containsAll(expected1.entrySet())); Map result2 = results.get(1).getMap(); - assertEquals(ImmutableMap.builder() + Map expected2 = ImmutableMap.builder() + .put("ti_nm", "name") + .put("ti_id", "order_id") + .put("e", "ti") + .put("co", EXPECTED_CONTEXTS) + .put("tna", "AF003") + .put("aid", "cloudfront") + .put("ti_cu", "currency") + .put("ttm", "456789") + .put("tz", "Etc/UTC") + .put("ti_pr", "1.0") + .put("ti_qu", "2") + .put("p", "srv") + .put("tv", Version.TRACKER) + .put("ti_ca", "category") + .put("ti_sk", "sku") + .build(); + + assertTrue(result2.entrySet().containsAll(expected2.entrySet())); + } + + @Test + public void testEcommerceTransactionItemAlone() throws InterruptedException { + // Although surprising, EcommerceTransactionItems are valid events and + // can be sent separately from EcommerceTransactions. + + tracker.track(EcommerceTransactionItem.builder() + .itemId("order_id") + .sku("sku") + .price(1.0) + .quantity(2) + .name("name") + .category("category") + .currency("currency") + .customContext(contexts) + .trueTimestamp(456789L) + .build()); + + // Then + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("ti_nm", "name") .put("ti_id", "order_id") .put("e", "ti") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("aid", "cloudfront") .put("ti_cu", "currency") - .put("dtm", "123456") .put("ttm", "456789") .put("tz", "Etc/UTC") .put("ti_pr", "1.0") @@ -155,99 +235,101 @@ public void testEcommerceEvent() { .put("tv", Version.TRACKER) .put("ti_ca", "category") .put("ti_sk", "sku") - .build(), result2); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testUnstructuredEventWithContext() { + public void testUnstructuredEventWithContext() throws InterruptedException { // When tracker.track(Unstructured.builder() .eventData(new SelfDescribingJson( - "payload", + "iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0", ImmutableMap.of("foo", "bar") )) .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("p", "srv") .put("tv", Version.TRACKER) .put("e", "ue") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("tz", "Etc/UTC") - .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"payload\",\"data\":{\"foo\":\"bar\"}}}") - .put("dtm", "123456") + .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0\",\"data\":{\"foo\":\"bar\"}}}") .put("ttm", "456789") .put("aid", "cloudfront") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testUnstructuredEventWithoutContext() { + public void testUnstructuredEventWithoutContext() throws InterruptedException { // When tracker.track(Unstructured.builder() .eventData(new SelfDescribingJson( - "payload", + "iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0", ImmutableMap.of("foo", "baær") )) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("p", "srv") .put("tv", Version.TRACKER) - .put("eid", EXPECTED_EVENT_ID) .put("e", "ue") .put("tna", "AF003") .put("tz", "Etc/UTC") - .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"payload\",\"data\":{\"foo\":\"baær\"}}}") - .put("dtm", "123456") + .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0\",\"data\":{\"foo\":\"baær\"}}}") .put("ttm", "456789") .put("aid", "cloudfront") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testUnstructuredEventWithoutTrueTimestamp() { + public void testUnstructuredEventWithoutTrueTimestamp() throws InterruptedException { // When tracker.track(Unstructured.builder() .eventData(new SelfDescribingJson( - "payload", - ImmutableMap.of("foo", "baær") + "iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0", + ImmutableMap.of("foo", "bar") )) - .deviceCreatedTimestamp(123456) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("p", "srv") .put("tv", Version.TRACKER) - .put("eid", EXPECTED_EVENT_ID) .put("e", "ue") .put("tna", "AF003") .put("tz", "Etc/UTC") - .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"payload\",\"data\":{\"foo\":\"baær\"}}}") - .put("dtm", "123456") + .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0\",\"data\":{\"foo\":\"bar\"}}}") .put("aid", "cloudfront") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testTrackPageView() { - tracker = new Tracker.TrackerBuilder(this.mockEmitter, "AF003", "cloudfront") + public void testTrackPageView() throws InterruptedException { + tracker = new Tracker.TrackerBuilder(mockEmitter, "AF003", "cloudfront") .subject(new Subject.SubjectBuilder().build()) .base64(false) .build(); @@ -259,15 +341,14 @@ public void testTrackPageView() { .pageTitle("title") .referrer("referer") .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() - .put("dtm", "123456") + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("ttm", "456789") .put("tz", "Etc/UTC") .put("e", "pv") @@ -275,159 +356,157 @@ public void testTrackPageView() { .put("tv", Version.TRACKER) .put("p", "srv") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("aid", "cloudfront") .put("refr", "referer") .put("url", "url") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testTrackTwoEvents() { + public void testTrackTwoEvents() throws InterruptedException { // When tracker.track(PageView.builder() .pageUrl("url") .pageTitle("title") .referrer("referer") - .deviceCreatedTimestamp(123456) - .trueTimestamp(456789L) - .eventId("9783090a-dace-4c85-a75c-933b4596a6c5") + .trueTimestamp(123456L) .build()); - tracker.track(PageView.builder() - .pageUrl("url") - .pageTitle("title") - .referrer("referer") - .deviceCreatedTimestamp(123456) + tracker.track(Unstructured.builder() + .eventData(new SelfDescribingJson( + "iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0", + ImmutableMap.of("foo", "bar") + )) .trueTimestamp(456789L) - .eventId("39139d43-ea13-4163-8559-adea258bf9c4") .build()); // Then - List results = mockEmitter.eventList; + Thread.sleep(500); + + List results = mockEmitter.eventList; assertEquals(2, results.size()); - Map result1 = results.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() - .put("dtm", "123456") - .put("ttm", "456789") + Map result1 = results.get(0).getMap(); + Map expected1 = ImmutableMap.builder() + .put("ttm", "123456") .put("tz", "Etc/UTC") .put("e", "pv") .put("page", "title") .put("tv", Version.TRACKER) .put("p", "srv") - .put("eid", "9783090a-dace-4c85-a75c-933b4596a6c5") .put("tna", "AF003") .put("aid", "cloudfront") .put("refr", "referer") .put("url", "url") - .build(), result1); + .build(); + + assertTrue(result1.entrySet().containsAll(expected1.entrySet())); - Map result2 = results.get(1).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() - .put("dtm", "123456") + Map result2 = results.get(1).getMap(); + Map expected2 = ImmutableMap.builder() .put("ttm", "456789") - .put("tz", "Etc/UTC") - .put("e", "pv") - .put("page", "title") - .put("tv", Version.TRACKER) .put("p", "srv") - .put("eid", "39139d43-ea13-4163-8559-adea258bf9c4") + .put("tv", Version.TRACKER) + .put("e", "ue") .put("tna", "AF003") + .put("tz", "Etc/UTC") + .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/example/jsonschema/1-0-0\",\"data\":{\"foo\":\"bar\"}}}") .put("aid", "cloudfront") - .put("refr", "referer") - .put("url", "url") - .build(), result2); + .build(); + + assertTrue(result2.entrySet().containsAll(expected2.entrySet())); } @Test - public void testTrackScreenView() { + public void testTrackScreenView() throws InterruptedException { // When tracker.track(ScreenView.builder() .name("name") .id("id") .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() - .put("dtm", "123456") + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("ttm", "456789") .put("tz", "Etc/UTC") .put("e", "ue") .put("tv", Version.TRACKER) .put("p", "srv") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("aid", "cloudfront") .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/screen_view/jsonschema/1-0-0\",\"data\":{\"id\":\"id\",\"name\":\"name\"}}}") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testTrackScreenViewWithTimestamp() { + public void testTrackScreenViewWithTimestamp() throws InterruptedException { // When tracker.track(ScreenView.builder() .name("name") .id("id") - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() - .put("dtm", "123456") + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("ttm", "456789") .put("tz", "Etc/UTC") .put("e", "ue") .put("tv", Version.TRACKER) .put("p", "srv") - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("aid", "cloudfront") .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/screen_view/jsonschema/1-0-0\",\"data\":{\"id\":\"id\",\"name\":\"name\"}}}") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testTrackScreenViewWithDefaultContextAndTimestamp() { + public void testTrackScreenViewWithDefaultContextAndTimestamp() throws InterruptedException { // When tracker.track(ScreenView.builder() .name("name") .id("id") .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("p", "srv") .put("tv", Version.TRACKER) .put("e", "ue") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("tz", "Etc/UTC") .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/screen_view/jsonschema/1-0-0\",\"data\":{\"id\":\"id\",\"name\":\"name\"}}}") - .put("dtm", "123456") .put("ttm", "456789") .put("aid", "cloudfront") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testTrackTiming() { + public void testTrackTiming() throws InterruptedException { // When tracker.track(Timing.builder() .category("category") @@ -435,30 +514,30 @@ public void testTrackTiming() { .variable("variable") .timing(10) .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("p", "srv") .put("tv", Version.TRACKER) .put("e", "ue") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("tz", "Etc/UTC") .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/timing/jsonschema/1-0-0\",\"data\":{\"category\":\"category\",\"label\":\"label\",\"timing\":10,\"variable\":\"variable\"}}}") - .put("dtm", "123456") .put("ttm", "456789") .put("aid", "cloudfront") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); } @Test - public void testTrackTimingWithSubject() { + public void testTrackTimingWithSubject() throws InterruptedException { // Make Subject Subject s1 = new Subject.SubjectBuilder().build(); s1.setIpAddress("127.0.0.1"); @@ -471,28 +550,29 @@ public void testTrackTimingWithSubject() { .variable("variable") .timing(10) .customContext(contexts) - .deviceCreatedTimestamp(123456) .trueTimestamp(456789L) - .eventId(EXPECTED_EVENT_ID) .subject(s1) .build()); // Then - Map result = mockEmitter.eventList.get(0).getTrackerPayloads().get(0).getMap(); - assertEquals(ImmutableMap.builder() + Thread.sleep(500); + + Map result = mockEmitter.eventList.get(0).getMap(); + Map expected = ImmutableMap.builder() .put("p", "srv") .put("ue_pr", "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow/timing/jsonschema/1-0-0\",\"data\":{\"category\":\"category\",\"label\":\"label\",\"timing\":10,\"variable\":\"variable\"}}}") .put("tv", Version.TRACKER) .put("e", "ue") .put("ip", "127.0.0.1") .put("co", EXPECTED_CONTEXTS) - .put("eid", EXPECTED_EVENT_ID) .put("tna", "AF003") .put("tz", "Etc/UTC") - .put("dtm", "123456") .put("ttm", "456789") .put("aid", "cloudfront") - .build(), result); + .build(); + + assertTrue(result.entrySet().containsAll(expected.entrySet())); + } // --- Tracker Setter & Getter Tests @@ -500,7 +580,7 @@ public void testTrackTimingWithSubject() { @Test public void testGetTrackerVersion() { Tracker tracker = new Tracker.TrackerBuilder(mockEmitter, "namespace", "an-app-id").build(); - assertEquals("java-0.10.1", tracker.getTrackerVersion()); + assertEquals("java-0.12.0", tracker.getTrackerVersion()); } @Test @@ -513,17 +593,23 @@ public void testSetDefaultPlatform() { @Test public void testSetSubject() { + // Subject objects always have timezone set TimeZone.setDefault(TimeZone.getTimeZone("Etc/UTC")); + Subject s1 = new Subject.SubjectBuilder().build(); + s1.setLanguage("EN"); Tracker tracker = new Tracker.TrackerBuilder(mockEmitter, "AF003", "cloudfront") .subject(s1) .build(); + Subject s2 = new Subject.SubjectBuilder().build(); s2.setColorDepth(24); tracker.setSubject(s2); + Map subjectPairs = new HashMap<>(); subjectPairs.put("tz", "Etc/UTC"); subjectPairs.put("cd", "24"); + assertEquals(subjectPairs, tracker.getSubject().getSubject()); } diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/UtilsTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/UtilsTest.java index 63906c5e..214ba30e 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/UtilsTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/UtilsTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterBuilderTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterBuilderTest.java index 2bb49acc..a1159ac4 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterBuilderTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterBuilderTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterTest.java index cbfb12aa..bd0bb05b 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/BatchEmitterTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -15,38 +15,38 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; import com.google.common.collect.Lists; +import com.snowplowanalytics.snowplow.tracker.constants.Parameter; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; - -import com.snowplowanalytics.snowplow.tracker.DevicePlatform; import com.snowplowanalytics.snowplow.tracker.payload.SelfDescribingJson; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerEvent; -import com.snowplowanalytics.snowplow.tracker.payload.TrackerParameters; import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; -import com.snowplowanalytics.snowplow.tracker.constants.Parameter; import com.snowplowanalytics.snowplow.tracker.events.PageView; import com.snowplowanalytics.snowplow.tracker.http.HttpClientAdapter; public class BatchEmitterTest { private MockHttpClientAdapter mockHttpClientAdapter; + private FlakyHttpClientAdapter flakyHttpClientAdapter; private BatchEmitter emitter; + // MockHttpClientAdapter always returns 200 public static class MockHttpClientAdapter implements HttpClientAdapter { public boolean isGetCalled = false; public boolean isPostCalled = false; + public int postCounter = 0; public SelfDescribingJson capturedPayload; @Override public int post(SelfDescribingJson payload) { isPostCalled = true; + postCounter++; capturedPayload = payload; return 200; } @@ -58,140 +58,172 @@ public int get(TrackerPayload payload) { } @Override - public String getUrl() { - return null; + public String getUrl() { return null; } + + @Override + public Object getHttpClient() { return null; } + } + + // this class fails to "send" the first 4 requests + // but returns a successful result (200) subsequently + static class FlakyHttpClientAdapter implements HttpClientAdapter { + int failedPostCounter = 0; + int successfulPostCounter = 0; + @Override + public int post(SelfDescribingJson payload) { + if (failedPostCounter >= 4) { + successfulPostCounter++; + return 200; + } + + failedPostCounter++; + return 500; } @Override - public Object getHttpClient() { - return null; + public int get(TrackerPayload payload) { return 0; } + + @Override + public String getUrl() { return null; } + + @Override + public Object getHttpClient() { return null; } + } + + // This class always returns failure code 403 + static class FailingHttpClientAdapter implements HttpClientAdapter { + int failedPostCounter = 0; + @Override + public int post(SelfDescribingJson payload) { + failedPostCounter++; + return 403; } + + @Override + public int get(TrackerPayload payload) { return 0; } + + @Override + public String getUrl() { return null; } + + @Override + public Object getHttpClient() { return null; } } @Before public void setUp() { mockHttpClientAdapter = new MockHttpClientAdapter(); + flakyHttpClientAdapter = new FlakyHttpClientAdapter(); emitter = BatchEmitter.builder() .httpClientAdapter(mockHttpClientAdapter) - .bufferSize(10) + .batchSize(10) .build(); } @Test public void addToBuffer_withLess10Payloads_shouldNotEmptyBuffer() throws InterruptedException { - // Given - List events = createEvents(2); - - // When - for (TrackerEvent event : events) { - emitter.emit(event); - } + TrackerPayload payload = createPayload(); + boolean result = emitter.add(payload); Thread.sleep(500); - // Then - Assert.assertFalse(mockHttpClientAdapter.isGetCalled); - - Assert.assertEquals(2, emitter.getBuffer().size()); - Assert.assertEquals(events, emitter.getBuffer()); + Assert.assertTrue(result); + Assert.assertFalse(mockHttpClientAdapter.isPostCalled); + Assert.assertEquals(1, emitter.getBuffer().size()); + Assert.assertEquals(payload, emitter.getBuffer().get(0)); } @Test public void addToBuffer_withMore10Payloads_shouldEmptyBuffer() throws InterruptedException { - // Given - List events = createEvents(10); - - // When - for (TrackerEvent event : events) { - emitter.emit(event); + List payloads = createPayloads(10); + for (TrackerPayload payload : payloads) { + emitter.add(payload); } Thread.sleep(500); - // Then Assert.assertTrue(mockHttpClientAdapter.isPostCalled); - @SuppressWarnings("unchecked") - List> capturedPayload = (List>) mockHttpClientAdapter.capturedPayload.getMap().get("data"); - - assertPayload(events, capturedPayload); - Assert.assertEquals(0, emitter.getBuffer().size()); + Assert.assertEquals(1, mockHttpClientAdapter.postCounter); } @Test - public void flushBuffer_shouldEmptyBuffer() throws InterruptedException { - // Given - List events = createEvents(2); + public void addToBuffer_doesNotAddEventIfBufferFull() { + emitter = BatchEmitter.builder() + .httpClientAdapter(mockHttpClientAdapter) + .bufferCapacity(1) + .build(); - // When - for (TrackerEvent event : events) { - emitter.emit(event); - } + emitter.add(createPayload()); + TrackerPayload differentPayload = createPayload(); + boolean result = emitter.add(differentPayload); + + Assert.assertFalse(emitter.getBuffer().contains(differentPayload)); + Assert.assertFalse(result); + } + + @Test + public void flushBuffer_shouldEmptyBuffer() throws InterruptedException { + List payloads = createPayloads(2); + for (TrackerPayload payload : payloads) { + emitter.add(payload); + } emitter.flushBuffer(); Thread.sleep(500); - // Then Assert.assertTrue(mockHttpClientAdapter.isPostCalled); - @SuppressWarnings("unchecked") List> capturedPayload = (List>) mockHttpClientAdapter.capturedPayload.getMap().get("data"); - assertPayload(events, capturedPayload); - + assertPayload(payloads, capturedPayload); Assert.assertEquals(0, emitter.getBuffer().size()); } @Test - public void setBufferSize_WithNegativeValue_ThrowInvalidArgumentException() { - Exception exception = Assert.assertThrows(IllegalArgumentException.class, () -> emitter.setBufferSize(-1)); - Assert.assertEquals("bufferSize must be greater than 0", exception.getMessage()); + public void setBatchSize_WithNegativeValue_ThrowInvalidArgumentException() { + Exception exception = Assert.assertThrows(IllegalArgumentException.class, () -> emitter.setBatchSize(-1)); + Assert.assertEquals("batchSize must be greater than 0", exception.getMessage()); } @Test - public void getFinalPost_shouldAddSTMParameter() throws InterruptedException { - // Given - List events = createEvents(10); + public void setAndGetBatchSizeWorksAsExpected() throws InterruptedException { + emitter.setBatchSize(2); + Assert.assertEquals(2, emitter.getBatchSize()); - // When - for (TrackerEvent event : events) { - emitter.emit(event); + List payloads = createPayloads(2); + for (TrackerPayload payload : payloads) { + emitter.add(payload); } Thread.sleep(500); - // Then Assert.assertTrue(mockHttpClientAdapter.isPostCalled); - - @SuppressWarnings("unchecked") - List> capturedPayload = (List>) mockHttpClientAdapter.capturedPayload.getMap().get("data"); - - for (Map payloadMap : capturedPayload) { - Assert.assertTrue(payloadMap.containsKey(Parameter.DEVICE_SENT_TIMESTAMP)); - } + Assert.assertEquals(0, emitter.getBuffer().size()); } @Test - public void emitterThreadFactory_correctlyNamesThreads() { - class MyRunnable implements Runnable { - @Override - public void run() {} + public void getFinalPost_shouldAddSTMParameter() throws InterruptedException { + List payloads = createPayloads(10); + for (TrackerPayload payload : payloads) { + emitter.add(payload); } - BatchEmitter.EmitterThreadFactory threadFactory = new BatchEmitter.EmitterThreadFactory(); - String threadName = threadFactory.newThread(new MyRunnable()).getName(); + Thread.sleep(500); - // It's pool-2 because pool-1 was created during emitter instantiation - Assert.assertEquals("snowplow-emitter-pool-2-request-thread-1", threadName); + Assert.assertTrue(mockHttpClientAdapter.isPostCalled); + @SuppressWarnings("unchecked") + List> capturedPayload = (List>) mockHttpClientAdapter.capturedPayload.getMap().get("data"); + + for (Map payloadMap : capturedPayload) { + Assert.assertTrue(payloadMap.containsKey(Parameter.DEVICE_SENT_TIMESTAMP)); + } } @Test public void threadsHaveExpectedNames() { - // A BufferConsumer thread is created on BatchEmitter instantiation. - // Calling flushBuffer() here to require another thread - causing - // creation of a request thread within the scheduledThreadPool. + // Calling flushBuffer() here to create a request thread for event sending emitter.flushBuffer(); // Create a list of all live thread names @@ -201,33 +233,148 @@ public void threadsHaveExpectedNames() { threadNames.add(thread.getName()); } - Assert.assertTrue(threadNames.contains("snowplow-emitter-BufferConsumer-thread-1")); - Assert.assertTrue(threadNames.contains("snowplow-emitter-pool-1-request-thread-1")); + // Because the threadpools are named by a static ThreadFactory, + // the pool number varies if this test is run in isolation or not + boolean matchResult = false; + for (String name : threadNames) { + if (Pattern.matches("snowplow-emitter-pool-\\d+-request-thread-1", name)) { + matchResult = true; + } + } + + Assert.assertTrue(matchResult); } - private List createEvents(int numEvents) { - final List payloads = Lists.newArrayList(); - for (int i = 0; i < numEvents; i++) { - payloads.add(createEvent()); + @Test + public void close_sendsEventsAndStopsThreads() throws InterruptedException { + List payloads = createPayloads(2); + for (TrackerPayload payload : payloads) { + emitter.add(payload); } - return payloads; + Thread.sleep(500); + + emitter.close(); + + Thread.sleep(500); + + // close() calls flushBuffer() to send all remaining stored events + Assert.assertTrue(mockHttpClientAdapter.isPostCalled); + Assert.assertEquals(0, emitter.getBuffer().size()); + + // these events can be added to storage but should not be sent + List morePayloads = createPayloads(20); + for (TrackerPayload payload : morePayloads) { + emitter.add(payload); + } + Assert.assertEquals(20, emitter.getBuffer().size()); } - private TrackerEvent createEvent() { + @Test + public void eventsThatFailToSendAreReturnedToEventBuffer() throws InterruptedException { + emitter = BatchEmitter.builder() + .httpClientAdapter(new FlakyHttpClientAdapter()) + .batchSize(10) + .build(); + + List payloads = createPayloads(2); + for (TrackerPayload payload : payloads) { + emitter.add(payload); + } + emitter.flushBuffer(); + Thread.sleep(500); + + List storedEvents = emitter.getBuffer(); + + Assert.assertEquals(2, storedEvents.size()); + Assert.assertTrue(storedEvents.contains(payloads.get(0))); + Assert.assertTrue(storedEvents.contains(payloads.get(1))); + } + + @Test + public void eventSendingFailureIncreasesBackoffTime() throws InterruptedException { + emitter = BatchEmitter.builder() + .httpClientAdapter(flakyHttpClientAdapter) + .batchSize(1) + .build(); + + List payloads = createPayloads(2); + for (TrackerPayload payload : payloads) { + emitter.add(payload); + } + Thread.sleep(500); + + Assert.assertEquals(100, emitter.getRetryDelay()); + } + + @Test + public void successfulSendAfterFailureResetsBackoffTime() throws InterruptedException { + // the FlakyHttpClientAdapter returns 500 for the first 4 requests + // then subsequently returns 200 + FlakyHttpClientAdapter flakyHttpClientAdapter = new FlakyHttpClientAdapter(); + emitter = BatchEmitter.builder() + .httpClientAdapter(flakyHttpClientAdapter) + .batchSize(1) + .threadCount(1) + .build(); + + List payloads = createPayloads(6); + for (TrackerPayload payload : payloads) { + emitter.add(payload); + } + + Thread.sleep(500); + + Assert.assertEquals(2, flakyHttpClientAdapter.successfulPostCounter); + Assert.assertEquals(0, emitter.getRetryDelay()); + } + + @Test + public void noRetryAfterDenylistResponseCode() throws InterruptedException { + List noRetry = new ArrayList<>(); + noRetry.add(403); + + // the FailingHttpClientAdapter always returns 403 + FailingHttpClientAdapter failingHttpClientAdapter = new FailingHttpClientAdapter(); + BatchEmitter emitter = BatchEmitter.builder() + .httpClientAdapter(failingHttpClientAdapter) + .batchSize(2) + .fatalResponseCodes(noRetry) + .build(); + + List payloads = createPayloads(4); + for (TrackerPayload payload : payloads) { + emitter.add(payload); + } + + Thread.sleep(500); + + Assert.assertEquals(2, failingHttpClientAdapter.failedPostCounter); + Assert.assertEquals(0, emitter.getRetryDelay()); + Assert.assertEquals(0, emitter.getBuffer().size()); + } + + private TrackerPayload createPayload() { PageView pv = PageView.builder() - .pageUrl("https://www.snowplowanalytics.com/") - .pageTitle("Snowplow") - .referrer("https://www.google.com/") - .build(); + .pageUrl("https://www.snowplowanalytics.com/") + .pageTitle("Snowplow") + .referrer("https://www.google.com/") + .build(); - return new TrackerEvent(pv, new TrackerParameters("appId", DevicePlatform.ServerSideApp, "namespace", "0.0.0", false), null); + return pv.getPayload(); } - private void assertPayload(List events, List> capturedPayload) { + private List createPayloads(int numPayloads) { + final List payloads = Lists.newArrayList(); + for (int i = 0; i < numPayloads; i++) { + payloads.add(createPayload()); + } + return payloads; + } + + private void assertPayload(List payloads, List> capturedPayload) { List> eventPayloads = new ArrayList<>(); - for (TrackerEvent event : events) { - //All PageView events so we can get(0) from payloads - eventPayloads.add(event.getTrackerPayloads().get(0).getMap()); + for (TrackerPayload payload : payloads) { + eventPayloads.add(payload.getMap()); } //Iterate through all captured payloads @@ -235,16 +382,17 @@ private void assertPayload(List events, List> boolean matchFound = false; for (Map eventMap : eventPayloads) { //Find the matching events - if (capturedMap.get("eid") == eventMap.get("eid")) { + if (Objects.equals(capturedMap.get("eid"), eventMap.get("eid"))) { matchFound = true; //Assert that all the entries in the event are in the captured payload //There might be extra entries in capturedMap, such as the STM parameter //check for these additional parameters in other tests - assertThat(eventMap.entrySet(), everyItem(is(in(capturedMap.entrySet())))); + Assert.assertTrue(capturedMap.entrySet().containsAll(eventMap.entrySet())); } } - assertThat(matchFound, is(true)); //Ensure every event was found + Assert.assertTrue(matchFound); //Ensure every event was found } } } + diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/InMemoryEventStoreTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/InMemoryEventStoreTest.java new file mode 100644 index 00000000..214a94d2 --- /dev/null +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/emitter/InMemoryEventStoreTest.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. + * + * This program is licensed to you under the Apache License Version 2.0, + * and you may not use this file except in compliance with the Apache License Version 2.0. + * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the Apache License Version 2.0 is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. + */ +package com.snowplowanalytics.snowplow.tracker.emitter; + +import com.snowplowanalytics.snowplow.tracker.events.PageView; +import com.snowplowanalytics.snowplow.tracker.payload.TrackerPayload; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +public class InMemoryEventStoreTest { + + private TrackerPayload trackerPayload; + private InMemoryEventStore eventStore; + + @Before + public void setUp() { + trackerPayload = createTrackerPayload(); + eventStore = new InMemoryEventStore(); + } + + @Test + public void correctlyAddAnEventToStore() { + boolean result = eventStore.addEvent(trackerPayload); + + Assert.assertTrue(result); + } + + @Test + public void getSize_returnsCorrectNumberOfStoredEvents() { + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + + Assert.assertEquals(2, eventStore.size()); + } + + @Test + public void getEventsFromStorage() { + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + + Assert.assertEquals(2, eventStore.getEventsBatch(2).getPayloads().size()); + Assert.assertEquals(2, eventStore.size()); + } + + @Test + public void doNotGetEventsIfFewerPresentThanAskedFor() throws NullPointerException { + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + + BatchPayload events = eventStore.getEventsBatch(3); + + Assert.assertNull(events); + } + + @Test + public void putEventsBackInBufferIfFailedToSend() { + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + eventStore.getEventsBatch(2); + + Assert.assertEquals(0, eventStore.size()); + + eventStore.cleanupAfterSendingAttempt(true, 1L); + + Assert.assertEquals(2, eventStore.size()); + } + + @Test + public void doNotPutEventsBackInBufferIfSent() { + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + eventStore.getEventsBatch(2); + + Assert.assertEquals(0, eventStore.size()); + + eventStore.cleanupAfterSendingAttempt(false, 1L); + + Assert.assertEquals(0, eventStore.size()); + } + + @Test + public void dropNewerEventsOnFailureWhenBufferFull() { + eventStore = new InMemoryEventStore(3); + + TrackerPayload differentPayload = createTrackerPayload(); + + eventStore.addEvent(differentPayload); + eventStore.getEventsBatch(1); + + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + eventStore.addEvent(trackerPayload); + + eventStore.cleanupAfterSendingAttempt(true, 1L); + Assert.assertEquals(3, eventStore.size()); + Assert.assertTrue(eventStore.getAllEvents().contains(differentPayload)); + + } + + private TrackerPayload createTrackerPayload() { + PageView pv = PageView.builder() + .pageUrl("https://www.snowplowanalytics.com/") + .pageTitle("Snowplow") + .referrer("https://www.google.com/") + .build(); + + return pv.getPayload(); + } +} diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapterTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapterTest.java index 4152c374..7e61050c 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapterTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/http/HttpClientAdapterTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -92,10 +92,15 @@ public void get_withSuccessfulStatusCode_isOk() throws Exception { data.add("space", "b a r"); adapter.get(data); + String eventId = data.getEventId(); + String dtm = Long.toString(data.getDeviceCreatedTimestamp()); + // Then assertEquals(1, mockWebServer.getRequestCount()); RecordedRequest recordedRequest = mockWebServer.takeRequest(); - assertEquals("/i?foo=bar&space=b%20a%20r", recordedRequest.getPath()); + + String expectedString = "/i?eid=" + eventId + "&dtm=" + dtm + "&foo=bar&space=b%20a%20r"; + assertEquals(expectedString, recordedRequest.getPath()); assertEquals("GET", recordedRequest.getMethod()); } diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJsonTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJsonTest.java index 1c831fac..2c763e94 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJsonTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/SelfDescribingJsonTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -47,8 +47,12 @@ public void testMakeSdjWithObject() { public void testMakeSdjWithTrackerPayload() { TrackerPayload data = new TrackerPayload(); data.add("value", "key"); + String eventId = data.getEventId(); + String dtm = Long.toString(data.getDeviceCreatedTimestamp()); + SelfDescribingJson sdj = new SelfDescribingJson("schema_string", data); - String expected = "{\"schema\":\"schema_string\",\"data\":{\"value\":\"key\"}}"; + + String expected = "{\"schema\":\"schema_string\",\"data\":{\"eid\":\"" + eventId + "\",\"dtm\":\"" + dtm + "\",\"value\":\"key\"}}"; String sdjString = sdj.toString(); assertNotNull(sdj); assertEquals(expected, sdjString); diff --git a/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayloadTest.java b/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayloadTest.java index a9ffb90c..6b2db9eb 100644 --- a/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayloadTest.java +++ b/src/test/java/com/snowplowanalytics/snowplow/tracker/payload/TrackerPayloadTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2021 Snowplow Analytics Ltd. All rights reserved. + * Copyright (c) 2014-2022 Snowplow Analytics Ltd. All rights reserved. * * This program is licensed to you under the Apache License Version 2.0, * and you may not use this file except in compliance with the Apache License Version 2.0. @@ -15,6 +15,7 @@ // Java import java.util.HashMap; import java.util.Map; +import java.util.UUID; // JUnit import org.junit.Test; @@ -23,6 +24,33 @@ public class TrackerPayloadTest { + @Test + public void testGetEventId() { + TrackerPayload payload = new TrackerPayload(); + + boolean isValidEventId = true; + try { + UUID.fromString(payload.getEventId()); + } catch (Exception e) { + isValidEventId = false; + } + + assertTrue(isValidEventId); + assertTrue(payload.getMap().containsKey("eid")); + assertEquals(payload.getEventId(), payload.getMap().get("eid")); + } + + @Test + public void testGetDeviceCreatedTimestamp() { + long currentTime = System.currentTimeMillis(); + TrackerPayload payload = new TrackerPayload(); + long timeDifference = payload.getDeviceCreatedTimestamp() - currentTime; + assertTrue(timeDifference < 1000); + + assertTrue(payload.getMap().containsKey("dtm")); + assertEquals(Long.toString(payload.getDeviceCreatedTimestamp()), payload.getMap().get("dtm")); + } + @Test public void testAddKeyValue() { TrackerPayload payload = new TrackerPayload();