diff --git a/benchmarks/Project.toml b/benchmarks/Project.toml index a8e8f09a2..25394e656 100644 --- a/benchmarks/Project.toml +++ b/benchmarks/Project.toml @@ -3,11 +3,17 @@ uuid = "d94a1522-c11e-44a7-981a-42bf5dc1a001" version = "0.1.0" [deps] +AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" DiffUtils = "8294860b-85a6-42f8-8c35-d911f667b5f6" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1" DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9" diff --git a/benchmarks/benchmark_body.jmd b/benchmarks/benchmark_body.jmd index 4e73f579e..ac52ced4d 100644 --- a/benchmarks/benchmark_body.jmd +++ b/benchmarks/benchmark_body.jmd @@ -1,14 +1,14 @@ ```julia -@time model_def(data)(); +@time model_def(data...)(); ``` ```julia -m = time_model_def(model_def, data); +m = time_model_def(model_def, data...); ``` ```julia suite = make_suite(m); -results = run(suite); +results = run(suite; seconds=WEAVE_ARGS[:seconds]); ``` ```julia @@ -19,11 +19,37 @@ results["evaluation_untyped"] results["evaluation_typed"] ``` +```julia +let k = "evaluation_simple_varinfo_nt" + haskey(results, k) && results[k] +end +``` + +```julia +let k = "evaluation_simple_varinfo_componentarray" + haskey(results, k) && results[k] +end +``` + +```julia +let k = "evaluation_simple_varinfo_dict" + haskey(results, k) && results[k] +end +``` + +```julia +let k = "evaluation_simple_varinfo_dict_from_nt" + haskey(results, k) && results[k] +end +``` + ```julia; echo=false; results="hidden"; -BenchmarkTools.save(joinpath("results", WEAVE_ARGS[:name], "$(nameof(m))_benchmarks.json"), results) +BenchmarkTools.save( + joinpath("results", WEAVE_ARGS[:name], "$(nameof(m))_benchmarks.json"), results +) ``` -```julia; wrap=false +```julia; wrap=false; echo=false if WEAVE_ARGS[:include_typed_code] typed = typed_code(m) end @@ -37,7 +63,7 @@ end ``` ```julia; wrap=false; echo=false; -if haskey(WEAVE_ARGS, :name_old) +if WEAVE_ARGS[:include_typed_code] && haskey(WEAVE_ARGS, :name_old) # We want to compare the generated code to the previous version. import DiffUtils typed_old = deserialize(joinpath("results", WEAVE_ARGS[:name_old], "$(nameof(m)).jls")); diff --git a/benchmarks/benchmarks.jmd b/benchmarks/benchmarks.jmd index 5b86b261e..19407b8dd 100644 --- a/benchmarks/benchmarks.jmd +++ b/benchmarks/benchmarks.jmd @@ -1,18 +1,25 @@ -# Benchmarks +`j display("text/markdown", "## $(WEAVE_ARGS[:name]) ##")` -## Setup +### Setup ```julia using BenchmarkTools, DynamicPPL, Distributions, Serialization ``` ```julia -import DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child +using DynamicPPLBenchmarks +using DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child ``` -## Models +### Environment -### `demo1` +```julia; echo=false; skip="notebook" +DynamicPPLBenchmarks.display_environment() +``` + +### Models + +#### `demo1` ```julia @model function demo1(x) @@ -23,17 +30,17 @@ import DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child end model_def = demo1; -data = 1.0; +data = (1.0,); ``` ```julia; results="markup"; echo=false -weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS) +weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS) ``` -### `demo2` +#### `demo2` ```julia -@model function demo2(y) +@model function demo2(y) # Our prior belief about the probability of heads in a coin. p ~ Beta(1, 1) @@ -43,17 +50,19 @@ weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS) # Heads or tails of a coin are drawn from a Bernoulli distribution. y[n] ~ Bernoulli(p) end + + return (; p) end model_def = demo2; -data = rand(0:1, 10); +data = (rand(0:1, 10),); ``` ```julia; results="markup"; echo=false -weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS) +weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS) ``` -### `demo3` +#### `demo3` ```julia @model function demo3(x) @@ -74,9 +83,10 @@ weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS) k = Vector{Int}(undef, N) for i in 1:N k[i] ~ Categorical(w) - x[:,i] ~ MvNormal([μ[k[i]], μ[k[i]]], 1.) + x[:, i] ~ MvNormal([μ[k[i]], μ[k[i]]], 1.0) end - return k + + return (; μ1, μ2, k) end model_def = demo3 @@ -88,26 +98,28 @@ N = 30 μs = [-3.5, 0.0] # Construct the data points. -data = mapreduce(c -> rand(MvNormal([μs[c], μs[c]], 1.), N), hcat, 1:2); +data = (mapreduce(c -> rand(MvNormal([μs[c], μs[c]], 1.0), N), hcat, 1:2),); ``` ```julia; echo=false -weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS) +weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS) ``` -### `demo4`: loads of indexing +#### `demo4`: lots of variables ```julia -@model function demo4(n, ::Type{TV}=Vector{Float64}) where {TV} +@model function demo4_1k(::Type{TV}=Vector{Float64}) where {TV} m ~ Normal() - x = TV(undef, n) + x = TV(undef, 1_000) for i in eachindex(x) x[i] ~ Normal(m, 1.0) end + + return (; m, x) end -model_def = demo4 -data = (100_000, ); +model_def = demo4_1k +data = (); ``` ```julia; echo=false @@ -115,16 +127,70 @@ weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS) ``` ```julia -@model function demo4_dotted(n, ::Type{TV}=Vector{Float64}) where {TV} +@model function demo4_10k(::Type{TV}=Vector{Float64}) where {TV} + m ~ Normal() + x = TV(undef, 10_000) + for i in eachindex(x) + x[i] ~ Normal(m, 1.0) + end + + return (; m, x) +end + +model_def = demo4_10k +data = (); +``` + +```julia; echo=false +weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS) +``` + +```julia +@model function demo4_100k(::Type{TV}=Vector{Float64}) where {TV} + m ~ Normal() + x = TV(undef, 100_000) + for i in eachindex(x) + x[i] ~ Normal(m, 1.0) + end + + return (; m, x) +end + +model_def = demo4_100k +data = (); +``` + +```julia; echo=false +weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS) +``` + +#### `demo4_dotted`: `.~` for large number of variables + +```julia +@model function demo4_100k_dotted(::Type{TV}=Vector{Float64}) where {TV} m ~ Normal() - x = TV(undef, n) + x = TV(undef, 100_000) x .~ Normal(m, 1.0) + + return (; m, x) end -model_def = demo4_dotted -data = (100_000, ); +model_def = demo4_100k_dotted +data = (); ``` ```julia; echo=false -weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS) +weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS) +``` + +```julia; echo=false +if haskey(WEAVE_ARGS, :name_old) + display(MIME"text/markdown"(), "## Comparison with $(WEAVE_ARGS[:name_old]) ##") +end +``` + +```julia; echo=false +if haskey(WEAVE_ARGS, :name_old) + DynamicPPLBenchmarks.judgementtable(WEAVE_ARGS[:name], WEAVE_ARGS[:name_old]) +end ``` diff --git a/benchmarks/src/DynamicPPLBenchmarks.jl b/benchmarks/src/DynamicPPLBenchmarks.jl index 362a8940f..8e66c28d9 100644 --- a/benchmarks/src/DynamicPPLBenchmarks.jl +++ b/benchmarks/src/DynamicPPLBenchmarks.jl @@ -2,6 +2,9 @@ module DynamicPPLBenchmarks using DynamicPPL using BenchmarkTools +using InteractiveUtils + +using ComponentArrays: ComponentArrays using Weave: Weave using Markdown: Markdown @@ -32,6 +35,44 @@ function benchmark_typed_varinfo!(suite, m) return suite end +function benchmark_simple_varinfo_namedtuple!(suite, m) + # We expect the model to return the random variables as a `NamedTuple`. + retvals = m() + + # Populate. + vi = SimpleVarInfo{Float64}(retvals) + vi_ca = SimpleVarInfo{Float64}(ComponentArrays.ComponentArray(retvals)) + + # Evaluate. + suite["evaluation_simple_varinfo_nt"] = @benchmarkable $m($vi, $(DefaultContext())) + suite["evaluation_simple_varinfo_componentarrays"] = @benchmarkable $m( + $vi_ca, $(DefaultContext()) + ) + return suite +end + +function benchmark_simple_varinfo_dict!(suite, m) + # Populate. + vi = SimpleVarInfo{Float64}(Dict()) + retvals = m(vi) + + # Evaluate. + suite["evaluation_simple_varinfo_dict"] = @benchmarkable $m($vi, $(DefaultContext())) + + # We expect the model to return the random variables as a `NamedTuple`. + vns = map(keys(retvals)) do k + VarName{k}() + end + vi = SimpleVarInfo{Float64}(Dict(zip(vns, values(retvals)))) + + # Evaluate. + suite["evaluation_simple_varinfo_dict_from_nt"] = @benchmarkable $m( + $vi, $(DefaultContext()) + ) + + return suite +end + function typed_code(m, vi=VarInfo(m)) rng = DynamicPPL.Random.MersenneTwister(42) spl = DynamicPPL.SampleFromPrior() @@ -51,6 +92,11 @@ function make_suite(model) benchmark_untyped_varinfo!(suite, model) benchmark_typed_varinfo!(suite, model) + if isdefined(DynamicPPL, :SimpleVarInfo) + benchmark_simple_varinfo_namedtuple!(suite, model) + benchmark_simple_varinfo_dict!(suite, model) + end + return suite end @@ -151,6 +197,7 @@ function weave_benchmarks( name=default_name(; include_commit_id=include_commit_id), name_old=nothing, include_typed_code=false, + seconds=10, doctype="github", outpath="results/$(name)/", kwargs..., @@ -159,6 +206,7 @@ function weave_benchmarks( :benchmarkbody => benchmarkbody, :name => name, :include_typed_code => include_typed_code, + :seconds => seconds, ) if !isnothing(name_old) args[:name_old] = name_old @@ -168,4 +216,36 @@ function weave_benchmarks( return Weave.weave(input, doctype; out_path=outpath, args=args, kwargs...) end +function display_environment() + display("text/markdown", "Computer Information:") + vinfo = sprint(InteractiveUtils.versioninfo) + display( + "text/markdown", + """ +``` +$(vinfo) +``` +""", + ) + + ctx = Pkg.API.Context() + + pkg_status = let io = IOBuffer() + Pkg.status(Pkg.API.Context(); io=io) + String(take!(io)) + end + + display( + "text/markdown", + """ +Package Information: +""", + ) + + md = "```\n$(pkg_status)\n```" + return display("text/markdown", md) +end + +include("tables.jl") + end # module diff --git a/benchmarks/src/tables.jl b/benchmarks/src/tables.jl new file mode 100644 index 000000000..6a5646212 --- /dev/null +++ b/benchmarks/src/tables.jl @@ -0,0 +1,234 @@ +using BenchmarkTools, Tables, PrettyTables, DrWatson + +####################### +# `TrialJudgementRow` # +####################### +struct TrialJudgementRow{names,Textras} <: Tables.AbstractRow + group::String + judgement::BenchmarkTools.TrialJudgement + extras::NamedTuple{names,Textras} +end + +function TrialJudgementRow(group::String, judgement::BenchmarkTools.TrialJudgement) + return TrialJudgementRow(group, judgement, NamedTuple()) +end + +function Tables.columnnames(::Type{TrialJudgementRow}) + return ( + :group, + :time, + :time_judgement, + :gctime, + :memory, + :memory_judgement, + :allocs, + :time_tolerance, + :memory_tolerance, + ) +end +# Dispatch needs to include all type-parameters because Tables.jl is a bit too aggressive +# when it comes to overloading this. +function Tables.columnnames(::Type{TrialJudgementRow{names,Textras}}) where {names,Textras} + return (Tables.columnnames(TrialJudgementRow)..., names...) +end +Tables.columnnames(row::TrialJudgementRow) = Tables.columnnames(typeof(row)) +function Tables.getcolumn(row::TrialJudgementRow, i::Int) + return Tables.getcolumn(row, Tables.columnnames(row)[i]) +end +function Tables.getcolumn(row::TrialJudgementRow, name::Symbol) + # NOTE: We need to use `getfield` below because `getproperty` is overloaded by Tables.jl + # and so we'll get a `StackOverflowError` if we try to do something like `row.group`. + return if name === :group + getfield(row, name) + elseif name in Tables.columnnames(TrialJudgementRow) + # `name` is one of the default columns + j = getfield(row, :judgement) + if name === :time_judgement + j.time + elseif name === :memory_judgement + j.memory + elseif name === :time_tolerance + params(j).time_tolerance + elseif name === :memory_tolerance + params(j).memory_tolerance + else + # Defer the rest to the `TrialRatio`. + r = j.ratio + getfield(r, name) + end + else + # One of `row.extras` + extras = getfield(row, :extras) + getfield(extras, name) + end +end + +Tables.istable(rows::Vector{<:TrialJudgementRow}) = true + +Tables.rows(rows::Vector{<:TrialJudgementRow}) = rows +Tables.rowaccess(rows::Vector{<:TrialJudgementRow}) = true + +# Because DataFrames.jl doesn't respect the `columnaccess`: +# https://github.com/JuliaData/DataFrames.jl/blob/2b9f6673547259bab9fb3bf3b5224eebc7b11ecd/src/other/tables.jl#L48-L61. +Tables.columnaccess(rows::Vector{<:TrialJudgementRow}) = true +function Tables.columns(rows::Vector{<:TrialJudgementRow}) + return (; + ((name, getproperty.(rows, name)) for name in Tables.columnnames(eltype(rows)))... + ) +end + +######################### +# PrettyTables.jl usage # +######################### +function make_highlighter_judgement(isgood) + function highlighter_judgement(data::Vector{<:TrialJudgementRow}, i, j) + names = Tables.columnnames(eltype(data)) + name = names[j] + row = data[i] + x = row[j] + + if name === :time || name === :time_judgement + j = row[:time_judgement] + if j === :improvement + return isgood + elseif j === :regression + return !isgood + end + elseif name === :memory || name === :memory_judgement + j = row[:memory_judgement] + if j === :improvement + return isgood + elseif j === :regression + return !isgood + end + end + + return false + end + + return highlighter_judgement +end + +function make_formatter(data::Vector{<:TrialJudgementRow}) + names = Tables.columnnames(eltype(data)) + function formatter_judgement(x, i, j) + name = names[j] + + if name in (:time, :memory, :allocs, :gctime) + return BenchmarkTools.prettydiff(x) + elseif name in (:time_tolerance, :memory_tolerance) + return BenchmarkTools.prettypercent(x) + end + + return x + end + + return formatter_judgement +end + +function Base.show(io::IO, ::MIME"text/plain", rows::Vector{<:TrialJudgementRow}) + hgood = Highlighter(make_highlighter_judgement(true); foreground=:green, bold=true) + hbad = Highlighter(make_highlighter_judgement(false); foreground=:red, bold=true) + formatter = make_formatter(rows) + return pretty_table(io, rows; highlighters=(hgood, hbad), formatters=(formatter,)) +end + +function Base.show(io::IO, ::MIME"text/html", rows::Vector{<:TrialJudgementRow}) + hgood = HTMLHighlighter( + make_highlighter_judgement(true), + HTMLDecoration(; color="green", font_weight="bold"), + ) + hbad = HTMLHighlighter( + make_highlighter_judgement(false), HTMLDecoration(; color="red", font_weight="bold") + ) + formatter = make_formatter(rows) + return pretty_table( + io, + rows; + backend=Val(:html), + highlighters=(hgood, hbad), + formatters=(formatter,), + tf=PrettyTables.tf_html_minimalist, + ) +end + +######################################################### +# Make it more convenient to load benchmarks into table # +######################################################### +function judgementtable( + results::AbstractVector, + results_old::AbstractVector, + extras=fill(NamedTuple(), length(results)); + stat=minimum, +) + @assert length(results_old) == length(results) "benchmarks have different lengths" + + return collect( + TrialJudgementRow( + groupname, + judge(stat(results[i][groupname]), stat(results_old[i][groupname])), + extras[i], + ) for i in eachindex(results) for + groupname in keys(results[i]) if groupname in keys(results_old[i]) + ) +end + +function judgementtable(name::String, name_old::String; kwargs...) + model_names = + map(filter(endswith("_benchmarks.json"), readdir(projectdir("results", name)))) do x + # Strip the suffix. + x[1:(end - 5)] + end + + results = [] + results_old = [] + for model_name in model_names + append!( + results, BenchmarkTools.load(projectdir("results", name, "$(model_name).json")) + ) + append!( + results_old, + BenchmarkTools.load(projectdir("results", name_old, "$(model_name).json")), + ) + end + + extras = [(model_name=model_name,) for model_name in model_names] + + return judgementtable(results, results_old, extras; kwargs...) +end + +function judgementtable_single( + results::AbstractVector, + reference_group::AbstractString, + extras=fill(NamedTuple(), length(results)); + stat=minimum, +) + return collect( + TrialJudgementRow( + groupname, + judge(stat(results[i][groupname]), stat(results[i][reference_group])), + extras[i], + ) for i in eachindex(results) for groupname in keys(results[i]) + ) +end + +function judgementtable_single( + name::AbstractString, reference_group::AbstractString; kwargs... +) + model_names = + map(filter(endswith("_benchmarks.json"), readdir(projectdir("results", name)))) do x + # Strip the suffix. + x[1:(end - 5)] + end + + results = [] + for model_name in model_names + append!( + results, BenchmarkTools.load(projectdir("results", name, "$(model_name).json")) + ) + end + + extras = [(model_name=model_name,) for model_name in model_names] + + return judgementtable_single(results, reference_group, extras; kwargs...) +end