add new Bonito based referenceupdater prototype

.github/workflows/reference_tests.yml

@@ -230,4 +230,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: n_missing_refimages
-          path: n_missing/
+          path: n_missing/

ReferenceTests/src/ReferenceTests.jl

@@ -38,8 +38,10 @@ using Images, FixedPointNumbers, Colors, ColorTypes
 
 include("database.jl")
 include("stable_rng.jl")
+include("compare_media.jl")
 include("runtests.jl")
 include("image_download.jl")
+include("cross_backend_scores.jl")
 
 export @include_reference_tests
 

ReferenceTests/src/compare_media.jl

@@ -0,0 +1,95 @@
+# NOTE: This file is reused by ReferenceUpdater
+
+function get_frames(a, b)
+    return (get_frames(a), get_frames(b))
+end
+
+rgbf_convert(x::AbstractMatrix{<:RGB}) = convert(Matrix{RGBf}, x)
+rgbf_convert(x::AbstractMatrix{<:RGBA}) = convert(Matrix{RGBAf}, x)
+
+if @isdefined Makie
+    using Makie: extract_frames
+else
+    # pulled from Makie so we don't need to include it
+    function extract_frames(video, frame_folder; loglevel="quiet")
+        path = joinpath(frame_folder, "frame%04d.png")
+        run(`$(FFMPEG_jll.ffmpeg()) -loglevel $(loglevel) -i $video -y $path`)
+    end
+end
+
+function get_frames(video::AbstractString)
+    mktempdir() do folder
+        afolder = joinpath(folder, "a")
+        mkpath(afolder)
+        extract_frames(video, afolder)
+        aframes = joinpath.(afolder, readdir(afolder))
+        if length(aframes) > 10
+            # we don't want to compare too many frames since it's time costly
+            # so we just compare 10 random frames if more than 10
+            samples = range(1, stop=length(aframes), length=10)
+            istep = round(Int, length(aframes) / 10)
+            samples = 1:istep:length(aframes)
+            aframes = aframes[samples]
+        end
+        return load.(aframes)
+    end
+end
+
+function compare_images(a::AbstractMatrix{<:Union{RGB,RGBA}}, b::AbstractMatrix{<:Union{RGB,RGBA}})
+
+    a = rgbf_convert(a)
+    b = rgbf_convert(b)
+
+    if size(a) != size(b)
+        @warn "images don't have the same size, difference will be Inf"
+        return Inf
+    end
+
+    approx_tile_size_px = 30
+
+    range_dim1 = round.(Int, range(0, size(a, 1), length = ceil(Int, size(a, 1) / approx_tile_size_px)))
+    range_dim2 = round.(Int, range(0, size(a, 2), length = ceil(Int, size(a, 2) / approx_tile_size_px)))
+
+    boundary_iter(boundaries) = zip(boundaries[1:end-1] .+ 1, boundaries[2:end])
+
+    _norm(rgb1::RGBf, rgb2::RGBf) = sqrt(sum(((rgb1.r - rgb2.r)^2, (rgb1.g - rgb2.g)^2, (rgb1.b - rgb2.b)^2)))
+    _norm(rgba1::RGBAf, rgba2::RGBAf) = sqrt(sum(((rgba1.r - rgba2.r)^2, (rgba1.g - rgba2.g)^2, (rgba1.b - rgba2.b)^2, (rgba1.alpha - rgba2.alpha)^2)))
+    _norm(c1::Colorant, c2::Colorant) = _norm(RGBAf(c1), RGBAf(c2))
+
+    # compute the difference score as the maximum of the mean squared differences over the color
+    # values of tiles over the image. using tiles is a simple way to increase the local sensitivity
+    # without directly going to pixel-based comparison
+    # it also makes the scores more comparable between reference images of different sizes, because the same
+    # local differences would be normed to different mean scores if the images have different numbers of pixels
+    return maximum(Iterators.product(boundary_iter(range_dim1), boundary_iter(range_dim2))) do ((mi1, ma1), (mi2, ma2))
+        @views mean(_norm.(a[mi1:ma1, mi2:ma2], b[mi1:ma1, mi2:ma2]))
+    end
+end
+
+function compare_media(a::AbstractString, b::AbstractString)
+    _, ext = splitext(a)
+    if ext in (".png", ".jpg", ".jpeg", ".JPEG", ".JPG")
+        imga = load(a)
+        imgb = load(b)
+        return compare_images(imga, imgb)
+    elseif ext in (".mp4", ".gif")
+        aframes = get_frames(a)
+        bframes = get_frames(b)
+        # Frames can differ in length, which usually shouldn't be the case but can happen
+        # when the implementation of record changes, or when the example changes its number of frames
+        # In that case, we just return inf + warn
+        if length(aframes) != length(bframes)
+            @warn "not the same number of frames in video, difference will be Inf"
+            return Inf
+        end
+        return maximum(compare_images.(aframes, bframes))
+    else
+        error("Unknown media extension: $ext")
+    end
+end
+
+function get_all_relative_filepaths_recursively(dir)
+    mapreduce(vcat, walkdir(dir)) do (root, dirs, files)
+        relpath.(joinpath.(root, files), dir)
+    end
+end

ReferenceTests/src/cross_backend_scores.jl

@@ -0,0 +1,35 @@
+function generate_backend_comparison_scores(root_folder)
+    ref = joinpath(root_folder, "GLMakie", "recorded", "GLMakie")
+    isdir(ref) || error("GLMakie subfolder $ref must exist")
+
+    for folder in readdir(root_folder)
+        isdir(joinpath(root_folder, folder)) || continue
+
+        if folder == "GLMakie" # create dummy file
+            close(open(joinpath(root_folder, folder, "cross_backend_scores.tsv"), "w"))
+            continue
+        end
+
+        @info "Generating comparison scores between $folder and GLMakie"
+        generate_backend_comparison_scores(joinpath(root_folder, folder, "recorded", folder), ref)
+    end
+
+    return
+end
+
+function generate_backend_comparison_scores(target_dir, reference_dir)
+    isdir(target_dir) || error("Invalid directory: $target_dir")
+    isdir(reference_dir) || error("Invalid directory: $reference_dir")
+
+    target_files = get_all_relative_filepaths_recursively(target_dir)
+
+    open(joinpath(target_dir, "../../cross_backend_scores.tsv"), "w") do file
+        for filepath in target_files
+            isfile(joinpath(reference_dir, filepath)) || continue
+            diff = compare_media(joinpath(target_dir, filepath),
+                                    joinpath(reference_dir, filepath))
+            println(file, diff, '\t', filepath)
+        end
+    end
+end
+

ReferenceTests/src/runtests.jl

@@ -1,86 +1,3 @@
-function get_frames(a, b)
-    return (get_frames(a), get_frames(b))
-end
-
-rgbf_convert(x::AbstractMatrix{<:RGB}) = convert(Matrix{RGBf}, x)
-rgbf_convert(x::AbstractMatrix{<:RGBA}) = convert(Matrix{RGBAf}, x)
-
-function get_frames(video::AbstractString)
-    mktempdir() do folder
-        afolder = joinpath(folder, "a")
-        mkpath(afolder)
-        Makie.extract_frames(video, afolder)
-        aframes = joinpath.(afolder, readdir(afolder))
-        if length(aframes) > 10
-            # we don't want to compare too many frames since it's time costly
-            # so we just compare 10 random frames if more than 10
-            samples = range(1, stop=length(aframes), length=10)
-            istep = round(Int, length(aframes) / 10)
-            samples = 1:istep:length(aframes)
-            aframes = aframes[samples]
-        end
-        return load.(aframes)
-    end
-end
-
-function compare_images(a::AbstractMatrix{<:Union{RGB,RGBA}}, b::AbstractMatrix{<:Union{RGB,RGBA}})
-
-    a = rgbf_convert(a)
-    b = rgbf_convert(b)
-
-    if size(a) != size(b)
-        @warn "images don't have the same size, difference will be Inf"
-        return Inf
-    end
-
-    approx_tile_size_px = 30
-
-    range_dim1 = round.(Int, range(0, size(a, 1), length = ceil(Int, size(a, 1) / approx_tile_size_px)))
-    range_dim2 = round.(Int, range(0, size(a, 2), length = ceil(Int, size(a, 2) / approx_tile_size_px)))
-
-    boundary_iter(boundaries) = zip(boundaries[1:end-1] .+ 1, boundaries[2:end])
-
-    _norm(rgb1::RGBf, rgb2::RGBf) = sqrt(sum(((rgb1.r - rgb2.r)^2, (rgb1.g - rgb2.g)^2, (rgb1.b - rgb2.b)^2)))
-    _norm(rgba1::RGBAf, rgba2::RGBAf) = sqrt(sum(((rgba1.r - rgba2.r)^2, (rgba1.g - rgba2.g)^2, (rgba1.b - rgba2.b)^2, (rgba1.alpha - rgba2.alpha)^2)))
-
-    # compute the difference score as the maximum of the mean squared differences over the color
-    # values of tiles over the image. using tiles is a simple way to increase the local sensitivity
-    # without directly going to pixel-based comparison
-    # it also makes the scores more comparable between reference images of different sizes, because the same
-    # local differences would be normed to different mean scores if the images have different numbers of pixels
-    return maximum(Iterators.product(boundary_iter(range_dim1), boundary_iter(range_dim2))) do ((mi1, ma1), (mi2, ma2))
-        @views mean(_norm.(a[mi1:ma1, mi2:ma2], b[mi1:ma1, mi2:ma2]))
-    end
-end
-
-function compare_media(a::AbstractString, b::AbstractString)
-    _, ext = splitext(a)
-    if ext in (".png", ".jpg", ".jpeg", ".JPEG", ".JPG")
-        imga = load(a)
-        imgb = load(b)
-        return compare_images(imga, imgb)
-    elseif ext in (".mp4", ".gif")
-        aframes = get_frames(a)
-        bframes = get_frames(b)
-        # Frames can differ in length, which usually shouldn't be the case but can happen
-        # when the implementation of record changes, or when the example changes its number of frames
-        # In that case, we just return inf + warn
-        if length(aframes) != length(bframes)
-            @warn "not the same number of frames in video, difference will be Inf"
-            return Inf
-        end
-        return maximum(compare_images.(aframes, bframes))
-    else
-        error("Unknown media extension: $ext")
-    end
-end
-
-function get_all_relative_filepaths_recursively(dir)
-    mapreduce(vcat, walkdir(dir)) do (root, dirs, files)
-        relpath.(joinpath.(root, files), dir)
-    end
-end
-
 function record_comparison(base_folder::String, backend::String; record_folder_name="recorded", tag=last_major_version())
     record_folder = joinpath(base_folder, record_folder_name)
     @info "Downloading reference images"
@@ -99,7 +16,7 @@ function record_comparison(base_folder::String, backend::String; record_folder_n
             println(file, path)
         end
     end
-    
+
     open(joinpath(base_folder, "missing_files.txt"), "w") do file
         backend_ref_dir = joinpath(reference_folder, backend)
         recorded_paths = mapreduce(vcat, walkdir(backend_ref_dir)) do (root, dirs, files)
@@ -134,8 +51,8 @@ function test_comparison(scores; threshold)
 end
 
 function compare(
-        relative_test_paths::Vector{String}, reference_dir::String, record_dir; 
-        o_refdir = reference_dir, missing_refimages = String[], 
+        relative_test_paths::Vector{String}, reference_dir::String, record_dir;
+        o_refdir = reference_dir, missing_refimages = String[],
         scores = Dict{String,Float64}()
     )
 

ReferenceTests/src/tests/examples3d.jl

@@ -598,7 +598,7 @@ end
         mesh!(ax, Rect2f(0.8, 0.1, 0.1, 0.8), space = :relative, color = :blue, shading = NoShading)
         linesegments!(ax, Rect2f(-0.5, -0.5, 1, 1), space = :clip, color = :cyan, linewidth = 5)
         text!(ax, 0, 0.52, text = "Clip Space", align = (:center, :bottom), space = :clip)
-        image!(ax, 0..40, 0..800, [x for x in range(0, 1, length=40), _ in 1:10], space = :pixel)
+        image!(ax, 0..40, 0..800, [x for x in range(0, 1, length=40), _ in 1:10], colormap = [:yellow, :green], space = :pixel)
     end
     fig
 end

ReferenceTests/src/tests/primitives.jl

@@ -610,7 +610,7 @@ function draw_marker_test!(scene, marker, center; markersize=300)
     scene
 end
 
-@reference_test "marke glyph alignment" begin
+@reference_test "marker glyph alignment" begin
     scene = Scene(size=(1200, 1200))
     campixel!(scene)
     # marker is in front, so it should not be smaller than the background rectangle

ReferenceUpdater/Project.toml

@@ -4,15 +4,27 @@ authors = ["Julius Krumbiegel <[email protected]>"]
 version = "0.1.0"
 
 [deps]
+Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8"
+Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+FFMPEG_jll = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
+FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
 ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]
+Bonito = "3, 4"
+Colors = "0.13.0"
+DelimitedFiles = "1.9.1"
+FFMPEG_jll = "6.1.2"
+FileIO = "1.16.6"
 HTTP = "1"
+Statistics = "1.11.1"

ReferenceUpdater/src/ReferenceUpdater.jl

@@ -22,6 +22,7 @@ end
 
 include("local_server.jl")
 include("image_download.jl")
+include("bonito-app.jl")
 
 basedir(files...) = normpath(joinpath(@__DIR__, "..", files...))