Skip to content

Commit

Permalink
More benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
@aherlihy
aherlihy committed Dec 2, 2023
1 parent c675d34 commit d87dade
Show file tree
Hide file tree
Showing 9 changed files with 552 additions and 319 deletions.
147 changes: 147 additions & 0 deletions bench/src/test/scala/datalog/benchmarks/BenchMacro.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@ import datalog.{
TastyslistlibinverseWorstMacroCompilerWithFacts as TastyslistlibinverseMacroCompilerWithFacts,
TastyslistlibinverseWorstMacroCompilerWithFactsOnline as TastyslistlibinverseMacroCompilerWithFactsOnline,
TastyslistlibinverseWorstMacroCompilerOnline as TastyslistlibinverseMacroCompilerOnline,
CbaexprvalueWorstMacroCompiler as CbaexprvalueMacroCompiler,
CbaexprvalueWorstMacroCompilerWithFacts as CbaexprvalueMacroCompilerWithFacts,
CbaexprvalueWorstMacroCompilerWithFactsOnline as CbaexprvalueMacroCompilerWithFactsOnline,
CbaexprvalueWorstMacroCompilerOnline as CbaexprvalueMacroCompilerOnline,
EqualWorstMacroCompiler as EqualMacroCompiler,
EqualWorstMacroCompilerWithFacts as EqualMacroCompilerWithFacts,
EqualWorstMacroCompilerWithFactsOnline as EqualMacroCompilerWithFactsOnline,
EqualWorstMacroCompilerOnline as EqualMacroCompilerOnline,
}

import scala.compiletime.uninitialized
Expand Down Expand Up @@ -54,6 +62,14 @@ object BenchMacro {
val tastyslistlibinverseWithFactsCompiled = TastyslistlibinverseMacroCompilerWithFacts.compile()
val tastyslistlibinverseOnlineCompiled = TastyslistlibinverseMacroCompilerOnline.compile()
val tastyslistlibinverseWithFactsOnlineCompiled = TastyslistlibinverseMacroCompilerWithFactsOnline.compile()
val equalCompiled = EqualMacroCompiler.compile()
val equalWithFactsCompiled = EqualMacroCompilerWithFacts.compile()
val equalOnlineCompiled = EqualMacroCompilerOnline.compile()
val equalWithFactsOnlineCompiled = EqualMacroCompilerWithFactsOnline.compile()
val cbaexprvalueCompiled = CbaexprvalueMacroCompiler.compile()
val cbaexprvalueWithFactsCompiled = CbaexprvalueMacroCompilerWithFacts.compile()
val cbaexprvalueOnlineCompiled = CbaexprvalueMacroCompilerOnline.compile()
val cbaexprvalueWithFactsOnlineCompiled = CbaexprvalueMacroCompilerWithFactsOnline.compile()
}
import BenchMacro.*

Expand Down Expand Up @@ -408,4 +424,135 @@ class BenchMacro {
// println(s"macro rules online, results =${res.size}")

}

/**
* Both facts + rules available at compile-time, no online optimization
*/
@Benchmark
def equal_macro_aot_offline(blackhole: Blackhole) = {
val expectedSize = 0
val facts = Paths.get(EqualMacroCompilerWithFacts.factDir)
val res = EqualMacroCompilerWithFacts.runCompiled(equalWithFactsCompiled)( // facts already loaded at compile-time
program => () //println(s"size succ = ${program.namedRelation("succ").get().size}")
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro AOT offline results =${res.size}")
}

/**
* Only rules available at compile-time, no online optimization
*/
@Benchmark
def equal_macro_rules_offline(blackhole: Blackhole) = {
val expectedSize = 0

val facts = Paths.get(EqualMacroCompiler.factDir)
val res = EqualMacroCompiler.runCompiled(equalCompiled)(
program =>
//println(s"size succ = ${program.namedRelation("succ").get().size}")
program.loadFromFactDir(facts.toString)
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro rules offline, results =${res.size}")
}

/**
* Both facts + rules available at compile-time, online optimization
*/
@Benchmark
def equal_macro_aot_online(blackhole: Blackhole) = {
val expectedSize = 0

val facts = Paths.get(EqualMacroCompilerWithFactsOnline.factDir)
val res = EqualMacroCompilerWithFactsOnline.runCompiled(equalWithFactsOnlineCompiled)(
program => {} // facts already loaded at compile-time
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro AOT online, results =${res.size}")
}

/**
* Only rules available at compile-time, online optimization
*/
@Benchmark
def equal_macro_rules_online(blackhole: Blackhole) = {
val expectedSize = 0

val facts = Paths.get(EqualMacroCompilerOnline.factDir)
val res = EqualMacroCompilerOnline.runCompiled(equalOnlineCompiled)(
program => program.loadFromFactDir(facts.toString)
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro rules online, results =${res.size}")
}

/**
* Both facts + rules available at compile-time, no online optimization
*/
@Benchmark
def cbaexprvalue_macro_aot_offline(blackhole: Blackhole) = {
val expectedSize = 6
val facts = Paths.get(CbaexprvalueMacroCompilerWithFacts.factDir)
val res = CbaexprvalueMacroCompilerWithFacts.runCompiled(cbaexprvalueWithFactsCompiled)( // facts already loaded at compile-time
program => () //println(s"size succ = ${program.namedRelation("succ").get().size}")
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro AOT offline results =${res.size}")
}

/**
* Only rules available at compile-time, no online optimization
*/
@Benchmark
def cbaexprvalue_macro_rules_offline(blackhole: Blackhole) = {
val expectedSize = 6

val facts = Paths.get(CbaexprvalueMacroCompiler.factDir)
val res = CbaexprvalueMacroCompiler.runCompiled(cbaexprvalueCompiled)(
program =>
//println(s"size succ = ${program.namedRelation("succ").get().size}")
program.loadFromFactDir(facts.toString)
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro rules offline, results =${res.size}")
}

/**
* Both facts + rules available at compile-time, online optimization
*/
@Benchmark
def cbaexprvalue_macro_aot_online(blackhole: Blackhole) = {
val expectedSize = 6

val facts = Paths.get(CbaexprvalueMacroCompilerWithFactsOnline.factDir)
val res = CbaexprvalueMacroCompilerWithFactsOnline.runCompiled(cbaexprvalueWithFactsOnlineCompiled)(
program => {} // facts already loaded at compile-time
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro AOT online, results =${res.size}")
}

/**
* Only rules available at compile-time, online optimization
*/
@Benchmark
def cbaexprvalue_macro_rules_online(blackhole: Blackhole) = {
val expectedSize = 6

val facts = Paths.get(CbaexprvalueMacroCompilerOnline.factDir)
val res = CbaexprvalueMacroCompilerOnline.runCompiled(cbaexprvalueOnlineCompiled)(
program => program.loadFromFactDir(facts.toString)
)
assert(res.size == expectedSize)
blackhole.consume(res)
// println(s"macro rules online, results =${res.size}")
}

}
138 changes: 85 additions & 53 deletions bench/src/test/scala/datalog/benchmarks/BenchMacroBaseline.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ import test.examples.tastyslistlib.tastyslistlib_worst
import test.examples.tastyslistlib.tastyslistlib_optimized
import test.examples.tastyslistlibinverse.tastyslistlibinverse_worst
import test.examples.tastyslistlibinverse.tastyslistlibinverse_optimized
import test.examples.cbaexprvalue.cbaexprvalue_worst
import test.examples.cbaexprvalue.cbaexprvalue_optimized
import test.examples.equal.equal_worst
import test.examples.equal.equal_optimized

object AckermannWorst extends ackermann_worst
object AckermannOptimized extends ackermann_optimized
Expand All @@ -33,6 +37,10 @@ object TastyslistlibWorst extends tastyslistlib_worst
object TastyslistlibOptimized extends tastyslistlib_optimized
object TastyslistlibinverseWorst extends tastyslistlibinverse_worst
object TastyslistlibinverseOptimized extends tastyslistlibinverse_optimized
object EqualWorst extends equal_worst
object EqualOptimized extends equal_optimized
object CbaexprvalueWorst extends cbaexprvalue_worst
object CbaexprvalueOptimized extends cbaexprvalue_optimized

@Fork(1)
@Warmup(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS, batchSize = 10)
Expand All @@ -54,18 +62,6 @@ class BenchMacroBaseline {
)
}

@Benchmark
def ackermann_jit_irop_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted, sortOrder = SortOrder.Sel)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(AckermannWorst.factDirectory)
AckermannWorst.pretest(program)

blackhole.consume(
program.namedRelation(AckermannWorst.toSolve).solve()
)
}

@Benchmark
def ackermann_interpreter_optimized_offline(blackhole: Blackhole) = {
Expand Down Expand Up @@ -107,18 +103,6 @@ class BenchMacroBaseline {
)
}

@Benchmark
def fib_jit_irop_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted, sortOrder = SortOrder.Sel)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(FibWorst.factDirectory)
FibWorst.pretest(program)

blackhole.consume(
program.namedRelation(FibWorst.toSolve).solve()
)
}

@Benchmark
def fib_interpreter_optimized_offline(blackhole: Blackhole) = {
Expand Down Expand Up @@ -160,9 +144,23 @@ class BenchMacroBaseline {
)
}


@Benchmark
def prime_jit_irop_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted, sortOrder = SortOrder.Sel)
def prime_interpreter_optimized_offline(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(PrimeOptimized.factDirectory)
PrimeOptimized.pretest(program)

blackhole.consume(
program.namedRelation(PrimeOptimized.toSolve).solve()
)
}

@Benchmark
def prime_interpreter_worst_offline(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(PrimeWorst.factDirectory)
Expand All @@ -173,49 +171,94 @@ class BenchMacroBaseline {
)
}

/** -----------------Cbaexprvalue-----------------* */
@Benchmark
def prime_interpreter_optimized_offline(blackhole: Blackhole) = {
def cbaexprvalue_jit_lambda_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.JIT, granularity = Granularity.DELTA, compileSync = CompileSync.Blocking, sortOrder = SortOrder.Sel, backend = Backend.Lambda)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(CbaexprvalueWorst.factDirectory)
CbaexprvalueWorst.pretest(program)

blackhole.consume(
program.namedRelation(CbaexprvalueWorst.toSolve).solve()
)
}


@Benchmark
def cbaexprvalue_interpreter_optimized_offline(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(PrimeOptimized.factDirectory)
PrimeOptimized.pretest(program)
program.loadFromFactDir(CbaexprvalueOptimized.factDirectory)
CbaexprvalueOptimized.pretest(program)

blackhole.consume(
program.namedRelation(PrimeOptimized.toSolve).solve()
program.namedRelation(CbaexprvalueOptimized.toSolve).solve()
)
}

@Benchmark
def prime_interpreter_worst_offline(blackhole: Blackhole) = {
def cbaexprvalue_interpreter_worst_offline(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(PrimeWorst.factDirectory)
PrimeWorst.pretest(program)
program.loadFromFactDir(CbaexprvalueWorst.factDirectory)
CbaexprvalueWorst.pretest(program)

blackhole.consume(
program.namedRelation(PrimeWorst.toSolve).solve()
program.namedRelation(CbaexprvalueWorst.toSolve).solve()
)
}

/** -----------------Tastyslistlib-----------------* */

/** -----------------Equal-----------------* */
@Benchmark
def tastyslistlib_jit_lambda_online(blackhole: Blackhole) = {
def equal_jit_lambda_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.JIT, granularity = Granularity.DELTA, compileSync = CompileSync.Blocking, sortOrder = SortOrder.Sel, backend = Backend.Lambda)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(TastyslistlibWorst.factDirectory)
TastyslistlibWorst.pretest(program)
program.loadFromFactDir(EqualWorst.factDirectory)
EqualWorst.pretest(program)

blackhole.consume(
program.namedRelation(TastyslistlibWorst.toSolve).solve()
program.namedRelation(EqualWorst.toSolve).solve()
)
}


@Benchmark
def equal_interpreter_optimized_offline(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(EqualOptimized.factDirectory)
EqualOptimized.pretest(program)

blackhole.consume(
program.namedRelation(EqualOptimized.toSolve).solve()
)
}

@Benchmark
def tastyslistlib_jit_irop_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted, sortOrder = SortOrder.Sel)
def equal_interpreter_worst_offline(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(EqualWorst.factDirectory)
EqualWorst.pretest(program)

blackhole.consume(
program.namedRelation(EqualWorst.toSolve).solve()
)
}


/** -----------------Tastyslistlib-----------------* */
@Benchmark
def tastyslistlib_jit_lambda_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.JIT, granularity = Granularity.DELTA, compileSync = CompileSync.Blocking, sortOrder = SortOrder.Sel, backend = Backend.Lambda)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(TastyslistlibWorst.factDirectory)
Expand All @@ -226,6 +269,7 @@ class BenchMacroBaseline {
)
}


@Benchmark
def tastyslistlib_interpreter_optimized_offline(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted)
Expand Down Expand Up @@ -253,18 +297,6 @@ class BenchMacroBaseline {
)
}

@Benchmark
def tastyslistlibinverse_jit_irop_online(blackhole: Blackhole) = {
val jo = JITOptions(mode = Mode.Interpreted, sortOrder = SortOrder.Sel)
val engine = new StagedExecutionEngine(new DefaultStorageManager(), jo)
val program = Program(engine)
program.loadFromFactDir(TastyslistlibinverseWorst.factDirectory)
TastyslistlibinverseWorst.pretest(program)

blackhole.consume(
program.namedRelation(TastyslistlibinverseWorst.toSolve).solve()
)
}

@Benchmark
def tastyslistlibinverse_interpreter_optimized_offline(blackhole: Blackhole) = {
Expand Down
4 changes: 3 additions & 1 deletion dl_bench.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ scp herlihy@diascld$1.iccluster.epfl.ch:/scratch/herlihy/carac/bench.out results

#cat results/$2$1.csv | sed -e 's/"datalog\.benchmarks\.examples\.\([^\.]*\)\./\1_/' | sed -e 's/_EOL\([^"]*\)"*/,/' | sed -e "1s/Benchmark/Benchmark_mode_storage_sort_onlineSort_fuzzy_compileSync_granularity_backend,/" | sed -e "1s/\"//g" > results/$2$1_clean.csv
cat results/$2$1.csv |
sed -e 's/"datalog\.benchmarks\.BenchMacro(.*)\.//' |
sed -e 's/"datalog\.benchmarks\.BenchMacroBaselineSlow\.//' |
sed -e 's/"datalog\.benchmarks\.BenchMacroBaseline\.//' |
sed -e 's/"datalog\.benchmarks\.BenchMacro\.//' |
sed -e 's/jit_lambda/jit-lambda_nothing/' |
sed -e 's/interpreter_optimized/interpreter-hand-optimized_nothing/' |
sed -e 's/interpreter_worst/interpreter_nothing/' |
Expand Down
Loading

0 comments on commit d87dade

Please sign in to comment.