Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .bazelversion
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6.4.0
17 changes: 1 addition & 16 deletions .ijwb/.bazelproject
Original file line number Diff line number Diff line change
@@ -1,16 +1 @@
directories:
# Add the directories you want added as source here
# By default, we've added your entire workspace ('.')
.

# Automatically includes all relevant targets under the 'directories' above
derive_targets_from_directories: true

targets:
# If source code isn't resolving, add additional targets that compile it here

additional_languages:
# Uncomment any additional languages you want supported
python
scala
java
import tools/ide_support/intellij/default_view.bazelproject
6 changes: 6 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,12 @@ load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")

scala_register_toolchains()

load("@io_bazel_rules_scala//testing:junit.bzl", "junit_repositories", "junit_toolchain")

junit_repositories()

junit_toolchain()

load("@io_bazel_rules_scala//testing:scalatest.bzl", "scalatest_repositories", "scalatest_toolchain")

scalatest_repositories()
Expand Down
4 changes: 2 additions & 2 deletions aggregator/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,13 @@ scala_library(
]),
)

scala_test_suite(
scala_junit_test_suite(
name = "test",
srcs = glob(["src/test/scala/ai/chronon/aggregator/test/*.scala"]),
suffixes = ["Test"],
visibility = ["//visibility:public"],
deps = [
":aggregator",
":test-lib",
"//api:api-lib",
"//api:api-models",
maven_artifact("junit:junit"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@ package ai.chronon.aggregator.test
import ai.chronon.aggregator.base.ApproxDistinctCount
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

class ApproxDistinctTest extends TestCase {

@Test
def testErrorBound(uniques: Int, errorBound: Int, lgK: Int): Unit = {
val uniqueElems = 1 to uniques
val duplicates = uniqueElems ++ uniqueElems ++ uniqueElems
Expand All @@ -32,6 +35,7 @@ class ApproxDistinctTest extends TestCase {
assertTrue(Math.abs(estimated - uniques) < errorBound)
}

@Test
def testMergingErrorBound(uniques: Int, errorBound: Int, lgK: Int, merges: Int): Unit = {
val chunkSize = uniques / merges
assert(chunkSize > 0)
Expand All @@ -50,12 +54,14 @@ class ApproxDistinctTest extends TestCase {
assertTrue(Math.abs(estimated - uniques) < errorBound)
}

@Test
def testErrorBounds(): Unit = {
testErrorBound(uniques = 100, errorBound = 1, lgK = 10)
testErrorBound(uniques = 1000, errorBound = 20, lgK = 10)
testErrorBound(uniques = 10000, errorBound = 300, lgK = 10)
}

@Test
def testMergingErrorBounds(): Unit = {
testMergingErrorBound(uniques = 100, errorBound = 1, lgK = 10, merges = 10)
testMergingErrorBound(uniques = 1000, errorBound = 20, lgK = 10, merges = 4)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package ai.chronon.aggregator.test
import ai.chronon.aggregator.base.{ApproxHistogram, ApproxHistogramIr}
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

import java.util
import scala.jdk.CollectionConverters._

class ApproxHistogramTest extends TestCase {

@Test
def testHistogram(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts = (1L to 3).map(i => i.toString -> i).toMap
Expand All @@ -18,6 +21,7 @@ class ApproxHistogramTest extends TestCase {
assertEquals(toHashMap(counts), approxHistogram.finalize(ir))
}

@Test
def testSketch(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts = (1L to 4).map(i => i.toString -> i).toMap
Expand All @@ -29,6 +33,7 @@ class ApproxHistogramTest extends TestCase {
assertEquals(toHashMap(expected), approxHistogram.finalize(ir))
}

@Test
def testMergeSketches(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts1: Map[String, Long] = Map("5" -> 5L, "4" -> 4, "2" -> 2, "1" -> 1)
Expand All @@ -51,6 +56,7 @@ class ApproxHistogramTest extends TestCase {
assertTrue(ir.histogram.isEmpty)
}

@Test
def testMergeHistograms(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts1: Map[String, Long] = Map("4" -> 4L, "2" -> 2)
Expand All @@ -73,6 +79,7 @@ class ApproxHistogramTest extends TestCase {
assertTrue(ir.sketch.isEmpty)
}

@Test
def testMergeHistogramsToSketch(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts1: Map[String, Long] = Map("4" -> 4L, "3" -> 3)
Expand All @@ -96,6 +103,7 @@ class ApproxHistogramTest extends TestCase {
assertTrue(ir.histogram.isEmpty)
}

@Test
def testMergeSketchAndHistogram(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts1: Map[String, Long] = Map("5" -> 5L, "3" -> 3, "2" -> 2, "1" -> 1)
Expand All @@ -118,6 +126,7 @@ class ApproxHistogramTest extends TestCase {
assert(ir.histogram.isEmpty)
}

@Test
def testNormalizeHistogram(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts = (1L to 3).map(i => i.toString -> i).toMap
Expand All @@ -128,6 +137,7 @@ class ApproxHistogramTest extends TestCase {
assertEquals(ir, normalized)
}

@Test
def testNormalizeSketch(): Unit = {
val approxHistogram = new ApproxHistogram[String](3)
val counts = (1L to 4).map(i => i.toString -> i).toMap
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@ import ai.chronon.aggregator.row.StatsGenerator
import com.yahoo.sketches.kll.KllFloatsSketch
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

import scala.util.Random

class ApproxPercentilesTest extends TestCase {
@transient lazy val logger = LoggerFactory.getLogger(getClass)

@Test
def testBasicImpl(nums: Int, slide: Int, k: Int, percentiles: Array[Double], errorPercent: Float): Unit = {
val sorted = (0 to nums).map(_.toFloat)
val elems = Random.shuffle(sorted.toList).toArray
Expand Down Expand Up @@ -54,6 +57,7 @@ class ApproxPercentilesTest extends TestCase {
diffs.foreach(diff => assertTrue(diff < errorMargin))
}

@Test
def testBasicPercentiles: Unit = {
val percentiles_tested: Int = 31
val percentiles: Array[Double] = (0 to percentiles_tested).toArray.map(i => i * 1.0 / percentiles_tested)
Expand All @@ -72,6 +76,7 @@ class ApproxPercentilesTest extends TestCase {
drift
}

@Test
def testPSIDrifts(): Unit = {
assertTrue(
getPSIDrift(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package ai.chronon.aggregator.test
import ai.chronon.aggregator.base.{FrequentItemType, FrequentItems, FrequentItemsFriendly, ItemsSketchIR}
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

import java.util
import scala.jdk.CollectionConverters._

class FrequentItemsTest extends TestCase {

@Test
def testNonPowerOfTwoAndTruncate(): Unit = {
val size = 3
val items = new FrequentItems[String](size)
Expand All @@ -31,6 +34,7 @@ class FrequentItemsTest extends TestCase {
result)
}

@Test
def testLessItemsThanSize(): Unit = {
val size = 10
val items = new FrequentItems[java.lang.Long](size)
Expand All @@ -53,6 +57,7 @@ class FrequentItemsTest extends TestCase {
result)
}

@Test
def testZeroSize(): Unit = {
val size = 0
val items = new FrequentItems[java.lang.Double](size)
Expand All @@ -69,6 +74,7 @@ class FrequentItemsTest extends TestCase {
assertEquals(new util.HashMap[String, Double](), result)
}

@Test
def testSketchSizes(): Unit = {
val expectedSketchSizes =
Map(
Expand All @@ -88,6 +94,7 @@ class FrequentItemsTest extends TestCase {
assertEquals(expectedSketchSizes, actualSketchSizes)
}

@Test
def testNormalization(): Unit = {
val testValues = (1 to 4)
.map(i => i -> i)
Expand Down Expand Up @@ -119,6 +126,7 @@ class FrequentItemsTest extends TestCase {
assertEquals(expectedStringValues, actualStringValues)
}

@Test
def testBulkMerge(): Unit = {
val sketch = new FrequentItems[String](3)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,14 @@ package ai.chronon.aggregator.test
import ai.chronon.aggregator.base.MinHeap
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

import java.util
import scala.collection.JavaConverters._

class MinHeapTest extends TestCase {

@Test
def testInserts(): Unit = {
val mh = new MinHeap[Int](maxSize = 4, Ordering.Int)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import ai.chronon.aggregator.base._
import junit.framework.TestCase
import org.apache.commons.math3.stat.descriptive.moment.{Kurtosis => ApacheKurtosis, Skewness => ApacheSkew}
import org.junit.Assert._
import org.junit.Test

class MomentTest extends TestCase {
def makeAgg(aggregator: MomentAggregator, values: Seq[Double]): (MomentAggregator, MomentsIR) = {
Expand Down Expand Up @@ -35,31 +36,36 @@ class MomentTest extends TestCase {
assertEquals(expected(v1 ++ v2), agg.finalize(ir), 0.1)
}

@Test
def testUpdate(): Unit = {
val values = Seq(1.1, 2.2, 3.3, 4.4, 5.5)
assertUpdate(new Skew(), values, expectedSkew)
assertUpdate(new Kurtosis(), values, expectedKurtosis)
}

@Test
def testInsufficientSizes(): Unit = {
val values = Seq(1.1, 2.2, 3.3, 4.4)
assertUpdate(new Skew(), values.take(2), _ => Double.NaN)
assertUpdate(new Kurtosis(), values.take(3), _ => Double.NaN)
}

@Test
def testNoVariance(): Unit = {
val values = Seq(1.0, 1.0, 1.0, 1.0)
assertUpdate(new Skew(), values, _ => Double.NaN)
assertUpdate(new Kurtosis(), values, _ => Double.NaN)
}

@Test
def testMerge(): Unit = {
val values1 = Seq(1.1, 2.2, 3.3)
val values2 = Seq(4.4, 5.5)
assertMerge(new Kurtosis(), values1, values2, expectedKurtosis)
assertMerge(new Skew(), values1, values2, expectedSkew)
}

@Test
def testNormalize(): Unit = {
val values = Seq(1.0, 2.0, 3.0, 4.0, 5.0)
val (agg, ir) = makeAgg(new Kurtosis, values)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import ai.chronon.aggregator.row.RowAggregator
import ai.chronon.api._
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

import java.util
import scala.collection.JavaConverters._
Expand Down Expand Up @@ -49,6 +50,7 @@ object TestRow {
}

class RowAggregatorTest extends TestCase {
@Test
def testUpdate(): Unit = {
val rows = List(
TestRow(1L, 4, 5.0f, "A", Seq(5, 3, 4), Seq("D", "A", "B", "A"), Map("A" -> 1, "B" -> 2)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import ai.chronon.api._
import com.google.gson.Gson
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

import java.util
import scala.collection.mutable
Expand All @@ -47,6 +48,7 @@ class Timer {

class SawtoothAggregatorTest extends TestCase {

@Test
def testTailAccuracy(): Unit = {
val timer = new Timer
val queries = CStream.genTimestamps(new Window(30, TimeUnit.DAYS), 10000, 5 * 60 * 1000)
Expand Down Expand Up @@ -118,6 +120,7 @@ class SawtoothAggregatorTest extends TestCase {
}
}

@Test
def testRealTimeAccuracy(): Unit = {
val timer = new Timer
val queries = CStream.genTimestamps(new Window(1, TimeUnit.DAYS), 1000)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@ import ai.chronon.api._
import com.google.gson.Gson
import junit.framework.TestCase
import org.junit.Assert.assertEquals
import org.junit.Test

import java.time.{Instant, ZoneOffset}
import java.time.format.DateTimeFormatter
import java.util.Locale

class SawtoothOnlineAggregatorTest extends TestCase {

@Test
def testConsistency(): Unit = {
val queryEndTs = TsUtils.round(System.currentTimeMillis(), WindowUtils.Day.millis)
val batchEndTs = queryEndTs - WindowUtils.Day.millis
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ import ai.chronon.aggregator.windowing.{TwoStackLiteAggregator, TwoStackLiteAggr
import ai.chronon.api.{Aggregation, Builders, IntType, LongType, Operation, StructField, StructType, TimeUnit, Window}
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test
import ai.chronon.api.Extensions.AggregationOps
import com.google.gson.Gson

import scala.collection.Seq

class TwoStackLiteAggregatorTest extends TestCase{
@Test
def testBufferWithTopK(): Unit = {
val topK = new TopK[Integer](IntType, 2)
val bankersBuffer = new TwoStackLiteAggregationBuffer(topK, 5)
Expand All @@ -53,6 +55,7 @@ class TwoStackLiteAggregatorTest extends TestCase{
assertBufferEquals(Seq(10), bankersBuffer.query)
}

@Test
def testAgainstSawtooth(): Unit = {
val timer = new Timer
val queries = CStream.genTimestamps(new Window(30, TimeUnit.DAYS), 100000, 5 * 60 * 1000)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory
import ai.chronon.aggregator.base.Variance
import junit.framework.TestCase
import org.junit.Assert._
import org.junit.Test

class VarianceTest extends TestCase {
@transient lazy val logger = LoggerFactory.getLogger(getClass)
Expand Down Expand Up @@ -59,6 +60,7 @@ class VarianceTest extends TestCase {
assertTrue((naiveResult - welfordResult) / naiveResult < 0.0000001)
}

@Test
def testVariance: Unit = {
compare(1000000)
compare(1000000, min = 100000, max = 100001)
Expand Down
Loading