From cbec66c9af49484d8263116148ecadea06dd3bba Mon Sep 17 00:00:00 2001
From: SimonDanisch <sdanisch@protonmail.com>
Date: Wed, 28 Feb 2024 11:53:41 +0100
Subject: [PATCH 01/26] add new Bonito based referenceupdater prototype

---
 ReferenceUpdater/src/bonito-app.jl | 68 ++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 ReferenceUpdater/src/bonito-app.jl

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
new file mode 100644
index 00000000000..6c279329431
--- /dev/null
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -0,0 +1,68 @@
+using Bonito, FileIO, DelimitedFiles
+
+folder = joinpath(pwd(), "ReferenceImages")
+
+App() do
+
+    scores_imgs = readdlm(joinpath(folder, "scores.tsv"), '\t')
+
+    scores = scores_imgs[:, 1]
+    imgs = scores_imgs[:, 2]
+    lookup = Dict(imgs .=> scores)
+
+    imgs_with_score = filter(x -> endswith(x, ".png"), unique(map(imgs) do img
+        replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
+    end))
+
+    function get_score(img_name)
+        backends = ["CairoMakie", "GLMakie", "WGLMakie"]
+        scores = map(backends) do backend
+            name = backend * "/" * img_name
+            if haskey(lookup, name)
+                return lookup[name]
+            else
+                return -Inf
+            end
+        end
+        return maximum(scores)
+    end
+
+    sort!(imgs_with_score; by=get_score, rev=true)
+
+    backends = ["CairoMakie", "GLMakie", "WGLMakie"]
+    images = map(imgs_with_score) do img_name
+        image_path = Observable{Any}(DOM.div())
+        path = Observable("recorded")
+        path_button = Bonito.Button("recorded")
+        on(path_button.value) do click
+            if path[] == "recorded"
+                path[] = "reference"
+                path_button.content[] = "reference"
+            else
+                path[] = "recorded"
+                path_button.content[] = "recorded"
+            end
+            return
+        end
+        buttons = map(backends) do backend
+            name = backend * "/" * img_name
+            if haskey(lookup, name)
+                score = round(lookup[name]; digits=4)
+                b = Bonito.Button("$backend: $score")
+                onany(b.value, path; update=true) do click, rec_ref
+                    bin = read(normpath(joinpath(folder, rec_ref, backend, img_name)))
+                    image_path[] = DOM.img(src=Bonito.BinaryAsset(bin, "image/png"))
+                end
+                return b
+            else
+                return DOM.div("$backend: X")
+            end
+        end
+        buttons = Row(path_button, buttons...; width="100%")
+
+        Card(Col(buttons, image_path); width="fit-content")
+    end
+
+    DOM.div(images...)
+
+end

From 2c70c89aa4d77cdabd6763eebacd7fc2b835eef4 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 14:43:50 +0100
Subject: [PATCH 02/26] update layout

---
 ReferenceUpdater/src/bonito-app.jl | 61 +++++++++++++++++-------------
 1 file changed, 34 insertions(+), 27 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 6c279329431..15d1791bf14 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -1,10 +1,10 @@
 using Bonito, FileIO, DelimitedFiles
 
-folder = joinpath(pwd(), "ReferenceImages")
+root_path = joinpath(pwd(), "ReferenceImages")
 
 App() do
 
-    scores_imgs = readdlm(joinpath(folder, "scores.tsv"), '\t')
+    scores_imgs = readdlm(joinpath(root_path, "scores.tsv"), '\t')
 
     scores = scores_imgs[:, 1]
     imgs = scores_imgs[:, 2]
@@ -30,37 +30,44 @@ App() do
     sort!(imgs_with_score; by=get_score, rev=true)
 
     backends = ["CairoMakie", "GLMakie", "WGLMakie"]
+    selected_folder = ["recorded", "reference"]
+    selection_string = ["Showing new recorded", "Showing old reference"]
+
+    button_style = Styles(
+        CSS("font-size" => "8", "font-weight" => "100"),
+        CSS("width" => "fit-content")
+    )
+
     images = map(imgs_with_score) do img_name
-        image_path = Observable{Any}(DOM.div())
-        path = Observable("recorded")
-        path_button = Bonito.Button("recorded")
-        on(path_button.value) do click
-            if path[] == "recorded"
-                path[] = "reference"
-                path_button.content[] = "reference"
-            else
-                path[] = "recorded"
-                path_button.content[] = "recorded"
-            end
-            return
-        end
-        buttons = map(backends) do backend
-            name = backend * "/" * img_name
-            if haskey(lookup, name)
-                score = round(lookup[name]; digits=4)
-                b = Bonito.Button("$backend: $score")
-                onany(b.value, path; update=true) do click, rec_ref
-                    bin = read(normpath(joinpath(folder, rec_ref, backend, img_name)))
-                    image_path[] = DOM.img(src=Bonito.BinaryAsset(bin, "image/png"))
+        cards = map(backends) do backend
+            # [] $path
+            # [Showing Reference/Recorded]  ---  Score: $score
+            # image
+
+            if haskey(lookup, backend * "/" * img_name)
+
+                path_button = Bonito.Button("recorded", style = button_style)
+                selection = 1 # Recorded (new), Reference (old)
+
+                image_element = map(path_button.value) do click
+                    selection = mod1(selection + 1, 2)
+                    path_button.content[] = selection_string[selection]
+                    folder = selected_folder[selection]
+
+                    bin = read(normpath(joinpath(root_path, folder, backend, img_name)))
+                    return DOM.img(src=Bonito.BinaryAsset(bin, "image/png"))
                 end
-                return b
+
+                # # score = round(lookup[name]; digits=4)
+                # # b = Bonito.Button("$backend: $score")
+
+                Card(Col(path_button, image_element))
             else
-                return DOM.div("$backend: X")
+                Card(DOM.h1("N/A"))
             end
         end
-        buttons = Row(path_button, buttons...; width="100%")
 
-        Card(Col(buttons, image_path); width="fit-content")
+        Grid(cards, columns = "1fr 1fr 1fr")
     end
 
     DOM.div(images...)

From 0e982c510c04a35b026eef60ece78b645566a721 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 15:17:13 +0100
Subject: [PATCH 03/26] add score text, checkbox

---
 ReferenceUpdater/src/bonito-app.jl | 49 ++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 12 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 15d1791bf14..301108fd7e2 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -10,6 +10,7 @@ App() do
     imgs = scores_imgs[:, 2]
     lookup = Dict(imgs .=> scores)
 
+    # TODO: don't filter out videos
     imgs_with_score = filter(x -> endswith(x, ".png"), unique(map(imgs) do img
         replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
     end))
@@ -33,20 +34,46 @@ App() do
     selected_folder = ["recorded", "reference"]
     selection_string = ["Showing new recorded", "Showing old reference"]
 
+    # TODO: font size doesn't do anything below some threshold?
+    # TODO: match up text & button styles
     button_style = Styles(
-        CSS("font-size" => "8", "font-weight" => "100"),
+        CSS("font-size" => "8", "font-weight" => "normal"),
         CSS("width" => "fit-content")
     )
 
+    # TODO: fit checkbox size to text
+    checkbox_style = Styles()
+
+    marked = Set{String}()
+
     images = map(imgs_with_score) do img_name
+
+        # [] $path
+        # [Showing Reference/Recorded]  ---  Score: $score # TODO:
+        # image
         cards = map(backends) do backend
-            # [] $path
-            # [Showing Reference/Recorded]  ---  Score: $score
-            # image
+            current_file = backend * "/" * img_name
+            if haskey(lookup, current_file)
+
+                local_marked = Observable(false)
+                on(local_marked) do is_marked
+                    if is_marked
+                        push!(marked, current_file)
+                    else
+                        delete!(marked, current_file)
+                    end
+                    @info marked
+                end
+                cb = DOM.div(
+                    Checkbox(local_marked, Dict{Symbol, Any}(:style => checkbox_style)),
+                    " $current_file"
+                )
 
-            if haskey(lookup, backend * "/" * img_name)
+                score = round(lookup[current_file]; digits=4)
+                score_text = DOM.div("Score: $score")
 
                 path_button = Bonito.Button("recorded", style = button_style)
+
                 selection = 1 # Recorded (new), Reference (old)
 
                 image_element = map(path_button.value) do click
@@ -58,18 +85,16 @@ App() do
                     return DOM.img(src=Bonito.BinaryAsset(bin, "image/png"))
                 end
 
-                # # score = round(lookup[name]; digits=4)
-                # # b = Bonito.Button("$backend: $score")
-
-                Card(Col(path_button, image_element))
+                # TODO: background
+                return Card(Col(cb, score_text, path_button, image_element))
             else
-                Card(DOM.h1("N/A"))
+                return Card(DOM.h1("N/A"))
             end
         end
 
-        Grid(cards, columns = "1fr 1fr 1fr")
+        return Grid(cards, columns = "1fr 1fr 1fr")
     end
 
-    DOM.div(images...)
+    return DOM.div(images...)
 
 end

From 64c346a7c9d08bca655794b7f76d6e60fa4552e8 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 16:51:57 +0100
Subject: [PATCH 04/26] force browser window

---
 ReferenceUpdater/src/bonito-app.jl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 301108fd7e2..b7e93aa1a3e 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -2,8 +2,8 @@ using Bonito, FileIO, DelimitedFiles
 
 root_path = joinpath(pwd(), "ReferenceImages")
 
-App() do
 
+function create_app()
     scores_imgs = readdlm(joinpath(root_path, "scores.tsv"), '\t')
 
     scores = scores_imgs[:, 1]
@@ -97,4 +97,10 @@ App() do
 
     return DOM.div(images...)
 
+
+if @isdefined server
+    close(server)
 end
+server = Bonito.Server("0.0.0.0", 8080)
+display(server)
+route!(server, "/" => App(create_app))

From f1ebdb656d786d7abab2e952e371999083678634 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 16:52:18 +0100
Subject: [PATCH 05/26] match order of master

---
 ReferenceUpdater/src/bonito-app.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index b7e93aa1a3e..aeeb9aba607 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -30,7 +30,7 @@ function create_app()
 
     sort!(imgs_with_score; by=get_score, rev=true)
 
-    backends = ["CairoMakie", "GLMakie", "WGLMakie"]
+    backends = ["GLMakie", "CairoMakie", "WGLMakie"]
     selected_folder = ["recorded", "reference"]
     selection_string = ["Showing new recorded", "Showing old reference"]
 

From 8305f61c2ed56fff969c32222b0aec496ce84953 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 16:53:43 +0100
Subject: [PATCH 06/26] prototype mp4 inclusion

---
 ReferenceUpdater/src/bonito-app.jl | 65 ++++++++++++++++++++++++++----
 1 file changed, 57 insertions(+), 8 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index aeeb9aba607..7c30404c6f9 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -11,9 +11,9 @@ function create_app()
     lookup = Dict(imgs .=> scores)
 
     # TODO: don't filter out videos
-    imgs_with_score = filter(x -> endswith(x, ".png"), unique(map(imgs) do img
+    imgs_with_score = unique(map(imgs) do img
         replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
-    end))
+    end)
 
     function get_score(img_name)
         backends = ["CairoMakie", "GLMakie", "WGLMakie"]
@@ -44,14 +44,27 @@ function create_app()
     # TODO: fit checkbox size to text
     checkbox_style = Styles()
 
+    # TODO: Is there a better way to handle default with overwrites?
+    card_css = CSS(
+        "margin" => "0.25em",
+        "padding" => "0.5em",
+        "border" => "2px solid lightblue",
+        # "background-color" => "#eee",
+        "border-radius" => "1em",
+    )
+
+    max_width = Styles(CSS("max-width" => "100%"))
+
     marked = Set{String}()
 
+    # TODO: one grid is probably better than a million single row grids...
     images = map(imgs_with_score) do img_name
 
         # [] $path
         # [Showing Reference/Recorded]  ---  Score: $score # TODO:
         # image
         cards = map(backends) do backend
+
             current_file = backend * "/" * img_name
             if haskey(lookup, current_file)
 
@@ -72,21 +85,57 @@ function create_app()
                 score = round(lookup[current_file]; digits=4)
                 score_text = DOM.div("Score: $score")
 
-                path_button = Bonito.Button("recorded", style = button_style)
+                card_style = Styles(card_css, CSS(
+                    "background-color" => if score > 0.05
+                        "#ffbbbb"
+                    elseif score > 0.03
+                        "#ffddbb"
+                    elseif score > 0.001
+                        "#ffffdd"
+                    else
+                        "#eeeeee"
+                    end
+                ))
 
+                path_button = Bonito.Button("recorded", style = button_style)
                 selection = 1 # Recorded (new), Reference (old)
-
-                image_element = map(path_button.value) do click
+                local_path = map(path_button.value) do click
                     selection = mod1(selection + 1, 2)
                     path_button.content[] = selection_string[selection]
                     folder = selected_folder[selection]
+                    return normpath(joinpath(root_path, folder, backend, img_name))
+                    # local_path = normpath(joinpath(root_path, folder, backend, img_name))
+                    # return Bonito.Asset(local_path)
+                end
 
-                    bin = read(normpath(joinpath(root_path, folder, backend, img_name)))
-                    return DOM.img(src=Bonito.BinaryAsset(bin, "image/png"))
+
+                filetype = split(img_name, ".")[end]
+                media = if filetype == "png"
+                    # DOM.img(src = local_path)
+                    map(local_path) do local_path
+                        bin = read(local_path)
+                        DOM.img(
+                            src = Bonito.BinaryAsset(bin, mimes[filetype]),
+                            style = max_width
+                        )
+                    end
+                else # TODO: broken
+                    # DOM.video(
+                    #     DOM.source(; src = local_path, type="video/mp4"),
+                    #     autoplay = true, controls = true
+                    # )
+                    asset = map(local_path) do p
+                        # Bonito.Asset(replace(p, ' ' => "\\ "))
+                        Bonito.Asset("\"$p\"")
+                    end
+                    DOM.video(
+                        DOM.source(; src = asset, type="video/mp4"),
+                        autoplay = true, controls = true
+                    )
                 end
 
                 # TODO: background
-                return Card(Col(cb, score_text, path_button, image_element))
+                return Card(Col(cb, score_text, path_button, media), style = card_style)
             else
                 return Card(DOM.h1("N/A"))
             end

From 4897ba30ecbdcd86532ba0a43ca6623ebf491ee6 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 16:56:13 +0100
Subject: [PATCH 07/26] prototype remaining sections

---
 ReferenceUpdater/src/bonito-app.jl | 41 ++++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 7c30404c6f9..4d61d8c1e91 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -115,7 +115,7 @@ function create_app()
                     map(local_path) do local_path
                         bin = read(local_path)
                         DOM.img(
-                            src = Bonito.BinaryAsset(bin, mimes[filetype]),
+                            src = Bonito.BinaryAsset(bin, "image/png"),
                             style = max_width
                         )
                     end
@@ -144,7 +144,44 @@ function create_app()
         return Grid(cards, columns = "1fr 1fr 1fr")
     end
 
-    return DOM.div(images...)
+    update_section = DOM.div(
+        DOM.h2("Images to update"),
+        Bonito.Button("Update reference images with selection"),
+        DOM.div("After pressing the button you will be asked which version to upload the reference images listed below to. After that the reference images on github will be replaced with an updated set if you have the rights to do so."),
+        DOM.h3("[TODO] images selected for updating:"),
+        DOM.div("TODO: image grid"),
+        DOM.h3("[TODO] images selected for removal:"),
+        DOM.div("TODO: image grid")
+    )
+
+    new_image_section = DOM.div(
+        DOM.h2("New images without references"),
+        DOM.div("The selected CI run produced an image for which no reference image exists. Selected images will be added as new reference images."),
+        DOM.div("TODO: toggle all"),
+        DOM.div("TODO: image grid")
+    )
+
+    missing_recordings_section = DOM.div(
+        DOM.h2("Old reference images without recordings"),
+        DOM.div("The selected CI run did not produce an image, but a reference image exists. This implies that a reference test was deleted or renamed. Selected images will be deleted from the reference images."),
+        DOM.div("TODO: toggle all"),
+        DOM.div("TODO: image grid")
+    )
+
+    main_section = DOM.div(
+        DOM.h2("Images with references"),
+        DOM.div("This is the normal case where the selected CI run produced an image and the reference image exists. Each row shows one image per backend from the same reference image test, which can be compared with its reference image. Rows are sorted based on the maximum row score (bigger = more different). Red cells fail CI (assuming the thresholds are up to date), yellow cells may but likely don't have significant visual difference and gray cells are visually equivalent."),
+        images...
+    )
+
+    return DOM.div(
+        update_section,
+        new_image_section,
+        missing_recordings_section,
+        main_section
+    )
+end
+
 
 
 if @isdefined server

From 21fff5b7770ad64031af6d63fff015876b0cf9fd Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 17:24:12 +0100
Subject: [PATCH 08/26] fix checkbox size, use one grid

---
 ReferenceUpdater/src/bonito-app.jl | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 4d61d8c1e91..470eb49f7d0 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -42,7 +42,9 @@ function create_app()
     )
 
     # TODO: fit checkbox size to text
-    checkbox_style = Styles()
+    checkbox_style = Styles(
+        CSS("transform" => "scale(1)")
+    )
 
     # TODO: Is there a better way to handle default with overwrites?
     card_css = CSS(
@@ -57,13 +59,13 @@ function create_app()
 
     marked = Set{String}()
 
-    # TODO: one grid is probably better than a million single row grids...
-    images = map(imgs_with_score) do img_name
+    cards = Any[]
+    for img_name in imgs_with_score
 
         # [] $path
         # [Showing Reference/Recorded]  ---  Score: $score # TODO:
         # image
-        cards = map(backends) do backend
+        for backend in backends
 
             current_file = backend * "/" * img_name
             if haskey(lookup, current_file)
@@ -82,9 +84,9 @@ function create_app()
                     " $current_file"
                 )
 
+
                 score = round(lookup[current_file]; digits=4)
                 score_text = DOM.div("Score: $score")
-
                 card_style = Styles(card_css, CSS(
                     "background-color" => if score > 0.05
                         "#ffbbbb"
@@ -98,7 +100,7 @@ function create_app()
                 ))
 
                 path_button = Bonito.Button("recorded", style = button_style)
-                selection = 1 # Recorded (new), Reference (old)
+                selection = 2 # Recorded (new), Reference (old)
                 local_path = map(path_button.value) do click
                     selection = mod1(selection + 1, 2)
                     path_button.content[] = selection_string[selection]
@@ -135,13 +137,13 @@ function create_app()
                 end
 
                 # TODO: background
-                return Card(Col(cb, score_text, path_button, media), style = card_style)
+                card = Card(Col(cb, score_text, path_button, media), style = card_style)
+                push!(cards, card)
             else
-                return Card(DOM.h1("N/A"))
+                push!(cards, Card(DOM.h1("N/A")))
             end
         end
 
-        return Grid(cards, columns = "1fr 1fr 1fr")
     end
 
     update_section = DOM.div(
@@ -171,14 +173,15 @@ function create_app()
     main_section = DOM.div(
         DOM.h2("Images with references"),
         DOM.div("This is the normal case where the selected CI run produced an image and the reference image exists. Each row shows one image per backend from the same reference image test, which can be compared with its reference image. Rows are sorted based on the maximum row score (bigger = more different). Red cells fail CI (assuming the thresholds are up to date), yellow cells may but likely don't have significant visual difference and gray cells are visually equivalent."),
-        images...
+        Grid(cards, columns = "1fr 1fr 1fr")
     )
 
     return DOM.div(
         update_section,
         new_image_section,
         missing_recordings_section,
-        main_section
+        main_section,
+        style = Styles(CSS("font-family" => "sans-serif"))
     )
 end
 

From ff4b03b891e3c1e63c06cd9c258a079df70fc03a Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 17:29:51 +0100
Subject: [PATCH 09/26] cleanup empty card, use Asset

---
 ReferenceUpdater/src/bonito-app.jl | 31 +++++++-----------------------
 1 file changed, 7 insertions(+), 24 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 470eb49f7d0..2a5e20246e3 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -105,42 +105,25 @@ function create_app()
                     selection = mod1(selection + 1, 2)
                     path_button.content[] = selection_string[selection]
                     folder = selected_folder[selection]
-                    return normpath(joinpath(root_path, folder, backend, img_name))
-                    # local_path = normpath(joinpath(root_path, folder, backend, img_name))
-                    # return Bonito.Asset(local_path)
+                    local_path = normpath(joinpath(root_path, folder, backend, img_name))
+                    return Bonito.Asset(local_path)
                 end
 
 
                 filetype = split(img_name, ".")[end]
                 media = if filetype == "png"
-                    # DOM.img(src = local_path)
-                    map(local_path) do local_path
-                        bin = read(local_path)
-                        DOM.img(
-                            src = Bonito.BinaryAsset(bin, "image/png"),
-                            style = max_width
-                        )
-                    end
-                else # TODO: broken
-                    # DOM.video(
-                    #     DOM.source(; src = local_path, type="video/mp4"),
-                    #     autoplay = true, controls = true
-                    # )
-                    asset = map(local_path) do p
-                        # Bonito.Asset(replace(p, ' ' => "\\ "))
-                        Bonito.Asset("\"$p\"")
-                    end
+                    DOM.img(src = local_path, style = max_width)
+                else
                     DOM.video(
-                        DOM.source(; src = asset, type="video/mp4"),
-                        autoplay = true, controls = true
+                        DOM.source(; src = local_path, type="video/mp4"),
+                        autoplay = false, controls = true, style = max_width
                     )
                 end
 
-                # TODO: background
                 card = Card(Col(cb, score_text, path_button, media), style = card_style)
                 push!(cards, card)
             else
-                push!(cards, Card(DOM.h1("N/A")))
+                push!(cards, DOM.div())
             end
         end
 

From 39fabae69b5636d6844310cce8a256f7184fa970 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 18:15:59 +0100
Subject: [PATCH 10/26] fix spacing, move score right of button

---
 ReferenceUpdater/src/bonito-app.jl | 37 ++++++++++++++++++++++++------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 2a5e20246e3..ea69cee2654 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -34,11 +34,24 @@ function create_app()
     selected_folder = ["recorded", "reference"]
     selection_string = ["Showing new recorded", "Showing old reference"]
 
-    # TODO: font size doesn't do anything below some threshold?
-    # TODO: match up text & button styles
+    # TODO: font size ignored?
     button_style = Styles(
-        CSS("font-size" => "8", "font-weight" => "normal"),
-        CSS("width" => "fit-content")
+        CSS("font-size" => "12", "font-weight" => "normal"),
+        CSS("width" => "fit-content",
+            "padding-right" => "6px", "padding-left" => "6px",
+            "padding-bottom" => "2px", "padding-top" => "2px",
+        ),
+        CSS("display" => "inline-block", "float" => "left")
+    )
+
+    score_style = Styles(
+        CSS("font-size" => "10", "font-weight" => "normal"),
+        CSS("width" => "fit-content",
+            "padding-right" => "6px", "padding-left" => "6px",
+            "padding-bottom" => "2px", "padding-top" => "2px",
+            "margin" => "0.25em"
+        ),
+        CSS("display" => "inline-block", "float" => "right")
     )
 
     # TODO: fit checkbox size to text
@@ -48,8 +61,8 @@ function create_app()
 
     # TODO: Is there a better way to handle default with overwrites?
     card_css = CSS(
-        "margin" => "0.25em",
-        "padding" => "0.5em",
+        "margin" => "0.1em", # outside
+        "padding" => "0.5em", # inside
         "border" => "2px solid lightblue",
         # "background-color" => "#eee",
         "border-radius" => "1em",
@@ -120,7 +133,17 @@ function create_app()
                     )
                 end
 
-                card = Card(Col(cb, score_text, path_button, media), style = card_style)
+                card = Card(
+                    DOM.div(
+                        cb,
+                        DOM.div(
+                            path_button,
+                            DOM.div("Score: $score", style = score_style)
+                        ),
+                        media
+                    ),
+                    style = card_style
+                )
                 push!(cards, card)
             else
                 push!(cards, DOM.div())

From 877037e4e3d5180ffd4beb59d47cd1e3867d568a Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 20:16:59 +0100
Subject: [PATCH 11/26] set up missing image grid

---
 ReferenceUpdater/src/bonito-app.jl | 177 +++++++++++++++++------------
 1 file changed, 107 insertions(+), 70 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index ea69cee2654..1a5a5b73bf7 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -4,35 +4,6 @@ root_path = joinpath(pwd(), "ReferenceImages")
 
 
 function create_app()
-    scores_imgs = readdlm(joinpath(root_path, "scores.tsv"), '\t')
-
-    scores = scores_imgs[:, 1]
-    imgs = scores_imgs[:, 2]
-    lookup = Dict(imgs .=> scores)
-
-    # TODO: don't filter out videos
-    imgs_with_score = unique(map(imgs) do img
-        replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
-    end)
-
-    function get_score(img_name)
-        backends = ["CairoMakie", "GLMakie", "WGLMakie"]
-        scores = map(backends) do backend
-            name = backend * "/" * img_name
-            if haskey(lookup, name)
-                return lookup[name]
-            else
-                return -Inf
-            end
-        end
-        return maximum(scores)
-    end
-
-    sort!(imgs_with_score; by=get_score, rev=true)
-
-    backends = ["GLMakie", "CairoMakie", "WGLMakie"]
-    selected_folder = ["recorded", "reference"]
-    selection_string = ["Showing new recorded", "Showing old reference"]
 
     # TODO: font size ignored?
     button_style = Styles(
@@ -70,42 +41,84 @@ function create_app()
 
     max_width = Styles(CSS("max-width" => "100%"))
 
-    marked = Set{String}()
 
-    cards = Any[]
-    for img_name in imgs_with_score
 
-        # [] $path
-        # [Showing Reference/Recorded]  ---  Score: $score # TODO:
-        # image
+    scores_imgs = readdlm(joinpath(root_path, "scores.tsv"), '\t')
+
+    scores = scores_imgs[:, 1]
+    imgs = scores_imgs[:, 2]
+    lookup = Dict(imgs .=> scores)
+
+    imgs_with_score = unique(map(imgs) do img
+        replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
+    end)
+
+    function get_score(img_name)
+        backends = ["CairoMakie", "GLMakie", "WGLMakie"]
+        scores = map(backends) do backend
+            name = backend * "/" * img_name
+            if haskey(lookup, name)
+                return lookup[name]
+            else
+                return -Inf
+            end
+        end
+        return maximum(scores)
+    end
+
+    function refimage_selection_checkbox(marked_set, current_file)
+        local_marked = Observable(false)
+        on(local_marked) do is_marked
+            if is_marked
+                push!(marked_set, current_file)
+            else
+                delete!(marked_set, current_file)
+            end
+            @info marked_set
+        end
+        return DOM.div(
+            Checkbox(local_marked, Dict{Symbol, Any}(:style => checkbox_style)),
+            " $current_file"
+        )
+    end
+
+    function media_element(img_name, local_path)
+        filetype = split(img_name, ".")[end]
+        if filetype == "png"
+            return DOM.img(src = local_path, style = max_width)
+        else
+            return DOM.video(
+                DOM.source(; src = local_path, type="video/mp4"),
+                autoplay = false, controls = true, style = max_width
+            )
+        end
+    end
+
+    sort!(imgs_with_score; by=get_score, rev=true)
+
+    backends = ["GLMakie", "CairoMakie", "WGLMakie"]
+    selected_folder = ["recorded", "reference"]
+    selection_string = ["Showing new recorded", "Showing old reference"]
+    score_thresholds = [0.05, 0.03, 0.01]
+
+    marked_for_update = Set{String}()
+
+    updated_cards = Any[]
+    for img_name in imgs_with_score
         for backend in backends
 
             current_file = backend * "/" * img_name
             if haskey(lookup, current_file)
 
-                local_marked = Observable(false)
-                on(local_marked) do is_marked
-                    if is_marked
-                        push!(marked, current_file)
-                    else
-                        delete!(marked, current_file)
-                    end
-                    @info marked
-                end
-                cb = DOM.div(
-                    Checkbox(local_marked, Dict{Symbol, Any}(:style => checkbox_style)),
-                    " $current_file"
-                )
-
+                cb = refimage_selection_checkbox(marked_for_update, current_file)
 
                 score = round(lookup[current_file]; digits=4)
-                score_text = DOM.div("Score: $score")
                 card_style = Styles(card_css, CSS(
-                    "background-color" => if score > 0.05
+                    "background-color" => if score > score_thresholds[1]
                         "#ffbbbb"
-                    elseif score > 0.03
+                    elseif score > score_thresholds[2]
                         "#ffddbb"
-                    elseif score > 0.001
+                    elseif score > score_thresholds[3]
                         "#ffffdd"
                     else
                         "#eeeeee"
@@ -122,16 +135,7 @@ function create_app()
                     return Bonito.Asset(local_path)
                 end
 
-
-                filetype = split(img_name, ".")[end]
-                media = if filetype == "png"
-                    DOM.img(src = local_path, style = max_width)
-                else
-                    DOM.video(
-                        DOM.source(; src = local_path, type="video/mp4"),
-                        autoplay = false, controls = true, style = max_width
-                    )
-                end
+                media = media_element(img_name, local_path)
 
                 card = Card(
                     DOM.div(
@@ -144,9 +148,41 @@ function create_app()
                     ),
                     style = card_style
                 )
-                push!(cards, card)
+                push!(updated_cards, card)
+            else
+                push!(updated_cards, DOM.div())
+            end
+        end
+
+    end
+
+    missing_files = readlines(joinpath(root_path, "missing_files.txt"))
+    missing_refimages = unique(map(missing_files) do img
+        replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
+    end)
+
+    marked_for_deletion = Set{String}()
+
+    missing_cards = Any[]
+    for img_name in missing_refimages
+        for backend in backends
+
+            current_file = backend * "/" * img_name
+            if current_file in missing_files
+
+                cb = refimage_selection_checkbox(marked_for_deletion, current_file)
+
+                local_path = map(path_button.value) do click
+                    local_path = normpath(joinpath(root_path, "reference", backend, img_name))
+                    return Bonito.Asset(local_path)
+                end
+
+                media = media_element(img_name, local_path)
+
+                card = Card(DOM.div(cb, media), style = card_style)
+                push!(missing_cards, card)
             else
-                push!(cards, DOM.div())
+                push!(missing_cards, DOM.div())
             end
         end
 
@@ -156,9 +192,9 @@ function create_app()
         DOM.h2("Images to update"),
         Bonito.Button("Update reference images with selection"),
         DOM.div("After pressing the button you will be asked which version to upload the reference images listed below to. After that the reference images on github will be replaced with an updated set if you have the rights to do so."),
-        DOM.h3("[TODO] images selected for updating:"),
+        DOM.h3("[TODO:] images selected for updating:"),
         DOM.div("TODO: image grid"),
-        DOM.h3("[TODO] images selected for removal:"),
+        DOM.h3("[TODO:] images selected for removal:"),
         DOM.div("TODO: image grid")
     )
 
@@ -173,13 +209,14 @@ function create_app()
         DOM.h2("Old reference images without recordings"),
         DOM.div("The selected CI run did not produce an image, but a reference image exists. This implies that a reference test was deleted or renamed. Selected images will be deleted from the reference images."),
         DOM.div("TODO: toggle all"),
-        DOM.div("TODO: image grid")
+        Grid(missing_cards, columns = "1fr 1fr 1fr")
     )
 
     main_section = DOM.div(
         DOM.h2("Images with references"),
-        DOM.div("This is the normal case where the selected CI run produced an image and the reference image exists. Each row shows one image per backend from the same reference image test, which can be compared with its reference image. Rows are sorted based on the maximum row score (bigger = more different). Red cells fail CI (assuming the thresholds are up to date), yellow cells may but likely don't have significant visual difference and gray cells are visually equivalent."),
-        Grid(cards, columns = "1fr 1fr 1fr")
+        DOM.div("This is the normal case where the selected CI run produced an image and the reference image exists. Each row shows one image per backend from the same reference image test, which can be compared with its reference image. Rows are sorted based on the maximum row score (bigger = more different). Background colors are based on this score, with red > $(score_thresholds[1]), orange > $(score_thresholds[2]), yellow > $(score_thresholds[3]) and the rest being light gray."),
+        DOM.br(),
+        Grid(updated_cards, columns = "1fr 1fr 1fr")
     )
 
     return DOM.div(

From 150a91bce76911d6279614aaa54355f862414cda Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 20:30:13 +0100
Subject: [PATCH 12/26] add new image grid & reorganize

---
 ReferenceUpdater/src/bonito-app.jl | 136 ++++++++++++++++-------------
 1 file changed, 74 insertions(+), 62 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 1a5a5b73bf7..3aae3bacc0a 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -5,6 +5,8 @@ root_path = joinpath(pwd(), "ReferenceImages")
 
 function create_app()
 
+    # Styles
+
     # TODO: font size ignored?
     button_style = Styles(
         CSS("font-size" => "12", "font-weight" => "normal"),
@@ -41,30 +43,13 @@ function create_app()
 
     max_width = Styles(CSS("max-width" => "100%"))
 
+    # Constants
 
-
-    scores_imgs = readdlm(joinpath(root_path, "scores.tsv"), '\t')
-
-    scores = scores_imgs[:, 1]
-    imgs = scores_imgs[:, 2]
-    lookup = Dict(imgs .=> scores)
-
-    imgs_with_score = unique(map(imgs) do img
-        replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
-    end)
-
-    function get_score(img_name)
-        backends = ["CairoMakie", "GLMakie", "WGLMakie"]
-        scores = map(backends) do backend
-            name = backend * "/" * img_name
-            if haskey(lookup, name)
-                return lookup[name]
-            else
-                return -Inf
-            end
-        end
-        return maximum(scores)
-    end
+    backends = ["GLMakie", "CairoMakie", "WGLMakie"]
+    # for updated images:
+    selected_folder = ["recorded", "reference"]
+    selection_string = ["Showing new recorded", "Showing old reference"]
+    score_thresholds = [0.05, 0.03, 0.01]
 
     function refimage_selection_checkbox(marked_set, current_file)
         local_marked = Observable(false)
@@ -94,14 +79,71 @@ function create_app()
         end
     end
 
-    sort!(imgs_with_score; by=get_score, rev=true)
+    function create_simple_grid_content(filename, image_folder)
+        files = readlines(joinpath(root_path, filename))
+        refimages = unique(map(files) do img
+            replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
+        end)
+
+        marked = Set{String}()
+
+        cards = Any[]
+        for img_name in refimages
+            for backend in backends
+
+                current_file = backend * "/" * img_name
+                if current_file in files
+                    cb = refimage_selection_checkbox(marked, current_file)
+                    local_path = Bonito.Asset(normpath(joinpath(root_path, image_folder, backend, img_name)))
+                    media = media_element(img_name, local_path)
+                    card = Card(DOM.div(cb, media), style = Styles(card_css))
+                    push!(cards, card)
+                else
+                    push!(cards, DOM.div())
+                end
+            end
 
-    backends = ["GLMakie", "CairoMakie", "WGLMakie"]
-    selected_folder = ["recorded", "reference"]
-    selection_string = ["Showing new recorded", "Showing old reference"]
-    score_thresholds = [0.05, 0.03, 0.01]
+        end
+
+        return cards, marked
+    end
+
+
+    # Newly added Images
+
+    new_cards, marked_for_upload = create_simple_grid_content("new_files.txt", "recorded")
+
+    # Deleted/Missing Images
+
+    missing_cards, marked_for_deletion = create_simple_grid_content("missing_files.txt", "reference")
+
+
+    # Updates images
 
-    marked_for_update = Set{String}()
+    scores_imgs = readdlm(joinpath(root_path, "scores.tsv"), '\t')
+
+    scores = scores_imgs[:, 1]
+    imgs = scores_imgs[:, 2]
+    lookup = Dict(imgs .=> scores)
+
+    imgs_with_score = unique(map(imgs) do img
+        replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
+    end)
+
+    function get_score(img_name)
+        backends = ["CairoMakie", "GLMakie", "WGLMakie"]
+        scores = map(backends) do backend
+            name = backend * "/" * img_name
+            if haskey(lookup, name)
+                return lookup[name]
+            else
+                return -Inf
+            end
+        end
+        return maximum(scores)
+    end
+
+    sort!(imgs_with_score; by=get_score, rev=true)
 
     updated_cards = Any[]
     for img_name in imgs_with_score
@@ -110,7 +152,7 @@ function create_app()
             current_file = backend * "/" * img_name
             if haskey(lookup, current_file)
 
-                cb = refimage_selection_checkbox(marked_for_update, current_file)
+                cb = refimage_selection_checkbox(marked_for_upload, current_file)
 
                 score = round(lookup[current_file]; digits=4)
                 card_style = Styles(card_css, CSS(
@@ -156,37 +198,7 @@ function create_app()
 
     end
 
-    missing_files = readlines(joinpath(root_path, "missing_files.txt"))
-    missing_refimages = unique(map(missing_files) do img
-        replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
-    end)
-
-    marked_for_deletion = Set{String}()
-
-    missing_cards = Any[]
-    for img_name in missing_refimages
-        for backend in backends
-
-            current_file = backend * "/" * img_name
-            if current_file in missing_files
-
-                cb = refimage_selection_checkbox(marked_for_deletion, current_file)
-
-                local_path = map(path_button.value) do click
-                    local_path = normpath(joinpath(root_path, "reference", backend, img_name))
-                    return Bonito.Asset(local_path)
-                end
-
-                media = media_element(img_name, local_path)
-
-                card = Card(DOM.div(cb, media), style = card_style)
-                push!(missing_cards, card)
-            else
-                push!(missing_cards, DOM.div())
-            end
-        end
-
-    end
+    # Create page
 
     update_section = DOM.div(
         DOM.h2("Images to update"),
@@ -202,7 +214,7 @@ function create_app()
         DOM.h2("New images without references"),
         DOM.div("The selected CI run produced an image for which no reference image exists. Selected images will be added as new reference images."),
         DOM.div("TODO: toggle all"),
-        DOM.div("TODO: image grid")
+        Grid(new_cards, columns = "1fr 1fr 1fr")
     )
 
     missing_recordings_section = DOM.div(

From b341a0ed839d5a8bf67fed256036ba99af3cbf31 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 20:43:47 +0100
Subject: [PATCH 13/26] set up counters

---
 ReferenceUpdater/src/bonito-app.jl | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 3aae3bacc0a..07a826a51d5 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -51,15 +51,16 @@ function create_app()
     selection_string = ["Showing new recorded", "Showing old reference"]
     score_thresholds = [0.05, 0.03, 0.01]
 
-    function refimage_selection_checkbox(marked_set, current_file)
+    function refimage_selection_checkbox(marked, current_file)
         local_marked = Observable(false)
         on(local_marked) do is_marked
             if is_marked
-                push!(marked_set, current_file)
+                push!(marked[], current_file)
             else
-                delete!(marked_set, current_file)
+                delete!(marked[], current_file)
             end
-            @info marked_set
+            notify(marked)
+            @info marked[]
         end
         return DOM.div(
             Checkbox(local_marked, Dict{Symbol, Any}(:style => checkbox_style)),
@@ -85,7 +86,7 @@ function create_app()
             replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
         end)
 
-        marked = Set{String}()
+        marked = Observable(Set{String}())
 
         cards = Any[]
         for img_name in refimages
@@ -204,9 +205,9 @@ function create_app()
         DOM.h2("Images to update"),
         Bonito.Button("Update reference images with selection"),
         DOM.div("After pressing the button you will be asked which version to upload the reference images listed below to. After that the reference images on github will be replaced with an updated set if you have the rights to do so."),
-        DOM.h3("[TODO:] images selected for updating:"),
+        DOM.h3(map(set -> "$(length(set)) images selected for updating:", marked_for_upload)),
         DOM.div("TODO: image grid"),
-        DOM.h3("[TODO:] images selected for removal:"),
+        DOM.h3(map(set -> "$(length(set)) images selected for removal:", marked_for_deletion)),
         DOM.div("TODO: image grid")
     )
 

From bc4e73a0a7cdc6740a3e2b73383db0cff6289b2c Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 20:55:32 +0100
Subject: [PATCH 14/26] toggle-all checkboxes

---
 ReferenceUpdater/src/bonito-app.jl | 32 ++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 07a826a51d5..18dbf449849 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -51,8 +51,12 @@ function create_app()
     selection_string = ["Showing new recorded", "Showing old reference"]
     score_thresholds = [0.05, 0.03, 0.01]
 
-    function refimage_selection_checkbox(marked, current_file)
-        local_marked = Observable(false)
+    function refimage_selection_checkbox(marked, current_file, mark_all = nothing)
+        if mark_all === nothing
+            local_marked = Observable(false)
+        else
+            local_marked = map(identity, mark_all)
+        end
         on(local_marked) do is_marked
             if is_marked
                 push!(marked[], current_file)
@@ -68,6 +72,14 @@ function create_app()
         )
     end
 
+    function check_all_checkbox()
+        checked = Observable(false) # maps to individual checkboxes which update marked
+        return DOM.div(
+            Checkbox(checked, Dict{Symbol, Any}(:style => checkbox_style)),
+            " Toggle All"
+        ), checked
+    end
+
     function media_element(img_name, local_path)
         filetype = split(img_name, ".")[end]
         if filetype == "png"
@@ -88,13 +100,15 @@ function create_app()
 
         marked = Observable(Set{String}())
 
+        toggle_all_cb, mark_all = check_all_checkbox()
+
         cards = Any[]
         for img_name in refimages
             for backend in backends
 
                 current_file = backend * "/" * img_name
                 if current_file in files
-                    cb = refimage_selection_checkbox(marked, current_file)
+                    cb = refimage_selection_checkbox(marked, current_file, mark_all)
                     local_path = Bonito.Asset(normpath(joinpath(root_path, image_folder, backend, img_name)))
                     media = media_element(img_name, local_path)
                     card = Card(DOM.div(cb, media), style = Styles(card_css))
@@ -106,17 +120,19 @@ function create_app()
 
         end
 
-        return cards, marked
+        return toggle_all_cb, cards, marked
     end
 
 
     # Newly added Images
 
-    new_cards, marked_for_upload = create_simple_grid_content("new_files.txt", "recorded")
+    new_checkbox, new_cards, marked_for_upload =
+        create_simple_grid_content("new_files.txt", "recorded")
 
     # Deleted/Missing Images
 
-    missing_cards, marked_for_deletion = create_simple_grid_content("missing_files.txt", "reference")
+    missing_checkbox, missing_cards, marked_for_deletion =
+        create_simple_grid_content("missing_files.txt", "reference")
 
 
     # Updates images
@@ -214,14 +230,14 @@ function create_app()
     new_image_section = DOM.div(
         DOM.h2("New images without references"),
         DOM.div("The selected CI run produced an image for which no reference image exists. Selected images will be added as new reference images."),
-        DOM.div("TODO: toggle all"),
+        new_checkbox,
         Grid(new_cards, columns = "1fr 1fr 1fr")
     )
 
     missing_recordings_section = DOM.div(
         DOM.h2("Old reference images without recordings"),
         DOM.div("The selected CI run did not produce an image, but a reference image exists. This implies that a reference test was deleted or renamed. Selected images will be deleted from the reference images."),
-        DOM.div("TODO: toggle all"),
+        missing_checkbox,
         Grid(missing_cards, columns = "1fr 1fr 1fr")
     )
 

From 91ba6c5286e4dd7bf030dacdaf5b9e93d202b9d6 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Wed, 22 Jan 2025 21:06:15 +0100
Subject: [PATCH 15/26] add update lists

---
 ReferenceUpdater/src/bonito-app.jl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 18dbf449849..156a1894fad 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -219,12 +219,12 @@ function create_app()
 
     update_section = DOM.div(
         DOM.h2("Images to update"),
-        Bonito.Button("Update reference images with selection"),
         DOM.div("After pressing the button you will be asked which version to upload the reference images listed below to. After that the reference images on github will be replaced with an updated set if you have the rights to do so."),
+        Bonito.Button("Update reference images with selection"),
         DOM.h3(map(set -> "$(length(set)) images selected for updating:", marked_for_upload)),
-        DOM.div("TODO: image grid"),
+        map(set -> DOM.ul([DOM.li(name) for name in set]), marked_for_upload),
         DOM.h3(map(set -> "$(length(set)) images selected for removal:", marked_for_deletion)),
-        DOM.div("TODO: image grid")
+        map(set -> DOM.ul([DOM.li(name) for name in set]), marked_for_deletion),
     )
 
     new_image_section = DOM.div(
@@ -264,4 +264,4 @@ if @isdefined server
 end
 server = Bonito.Server("0.0.0.0", 8080)
 display(server)
-route!(server, "/" => App(create_app))
+route!(server, "/" => App(create_app))
\ No newline at end of file

From c842f9e5e14c72d00c1efe8cdcbad4e0b6673afa Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 13:55:08 +0100
Subject: [PATCH 16/26] fix typo and tweak refimg to test functionality

---
 ReferenceTests/src/tests/examples3d.jl | 2 +-
 ReferenceTests/src/tests/primitives.jl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ReferenceTests/src/tests/examples3d.jl b/ReferenceTests/src/tests/examples3d.jl
index 63d44b3561a..b91a15dd7f8 100644
--- a/ReferenceTests/src/tests/examples3d.jl
+++ b/ReferenceTests/src/tests/examples3d.jl
@@ -587,7 +587,7 @@ end
         mesh!(ax, Rect2f(0.8, 0.1, 0.1, 0.8), space = :relative, color = :blue, shading = NoShading)
         linesegments!(ax, Rect2f(-0.5, -0.5, 1, 1), space = :clip, color = :cyan, linewidth = 5)
         text!(ax, 0, 0.52, text = "Clip Space", align = (:center, :bottom), space = :clip)
-        image!(ax, 0..40, 0..800, [x for x in range(0, 1, length=40), _ in 1:10], space = :pixel)
+        image!(ax, 0..40, 0..800, [x for x in range(0, 1, length=40), _ in 1:10], colormap = [:yellow, :green], space = :pixel)
     end
     fig
 end
diff --git a/ReferenceTests/src/tests/primitives.jl b/ReferenceTests/src/tests/primitives.jl
index 16b063e5aaa..63d22897fb7 100644
--- a/ReferenceTests/src/tests/primitives.jl
+++ b/ReferenceTests/src/tests/primitives.jl
@@ -610,7 +610,7 @@ function draw_marker_test!(scene, marker, center; markersize=300)
     scene
 end
 
-@reference_test "marke glyph alignment" begin
+@reference_test "marker glyph alignment" begin
     scene = Scene(size=(1200, 1200))
     campixel!(scene)
     # marker is in front, so it should not be smaller than the background rectangle

From aedf158c79ab6af29e89411e206fabe3b2fbbf1d Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 15:07:50 +0100
Subject: [PATCH 17/26] add app to ReferenceUpdater & hook up upload

---
 ReferenceUpdater/Project.toml            |   4 +
 ReferenceUpdater/src/ReferenceUpdater.jl |   1 +
 ReferenceUpdater/src/bonito-app.jl       | 101 +++++++++++++++++------
 3 files changed, 82 insertions(+), 24 deletions(-)

diff --git a/ReferenceUpdater/Project.toml b/ReferenceUpdater/Project.toml
index 59313979845..9578e87af8f 100644
--- a/ReferenceUpdater/Project.toml
+++ b/ReferenceUpdater/Project.toml
@@ -4,7 +4,9 @@ authors = ["Julius Krumbiegel <julius.krumbiegel@gmail.com>"]
 version = "0.1.0"
 
 [deps]
+Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
@@ -15,4 +17,6 @@ ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
 ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]
+Bonito = "3, 4"
+DelimitedFiles = "1.9.1"
 HTTP = "1"
diff --git a/ReferenceUpdater/src/ReferenceUpdater.jl b/ReferenceUpdater/src/ReferenceUpdater.jl
index bca0ba7513a..4e0265c351e 100644
--- a/ReferenceUpdater/src/ReferenceUpdater.jl
+++ b/ReferenceUpdater/src/ReferenceUpdater.jl
@@ -22,6 +22,7 @@ end
 
 include("local_server.jl")
 include("image_download.jl")
+include("bonito-app.jl")
 
 basedir(files...) = normpath(joinpath(@__DIR__, "..", files...))
 
diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 156a1894fad..b924234cbd7 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -1,9 +1,10 @@
-using Bonito, FileIO, DelimitedFiles
+using Bonito, DelimitedFiles
 
-root_path = joinpath(pwd(), "ReferenceImages")
 
-
-function create_app()
+"""
+    create_app(path_to_reference_images_folder)
+"""
+function create_app_content(root_path::String)
 
     # Styles
 
@@ -27,18 +28,15 @@ function create_app()
         CSS("display" => "inline-block", "float" => "right")
     )
 
-    # TODO: fit checkbox size to text
-    checkbox_style = Styles(
-        CSS("transform" => "scale(1)")
-    )
+    checkbox_style = Styles(CSS("transform" => "scale(1)"))
 
-    # TODO: Is there a better way to handle default with overwrites?
     card_css = CSS(
         "margin" => "0.1em", # outside
         "padding" => "0.5em", # inside
         "border" => "2px solid lightblue",
-        # "background-color" => "#eee",
         "border-radius" => "1em",
+        "color" => "black",
+        "min-width" => "16rem" # effectively controls size of the whole layout
     )
 
     max_width = Styles(CSS("max-width" => "100%"))
@@ -111,7 +109,7 @@ function create_app()
                     cb = refimage_selection_checkbox(marked, current_file, mark_all)
                     local_path = Bonito.Asset(normpath(joinpath(root_path, image_folder, backend, img_name)))
                     media = media_element(img_name, local_path)
-                    card = Card(DOM.div(cb, media), style = Styles(card_css))
+                    card = Card(DOM.div(cb, media), style = Styles(card_css, CSS("background-color" => "#eeeeee")))
                     push!(cards, card)
                 else
                     push!(cards, DOM.div())
@@ -171,7 +169,8 @@ function create_app()
 
                 cb = refimage_selection_checkbox(marked_for_upload, current_file)
 
-                score = round(lookup[current_file]; digits=4)
+                score = round(lookup[current_file]; digits=3)
+                # TODO: Is there a better way to handle default with overwrites?
                 card_style = Styles(card_css, CSS(
                     "background-color" => if score > score_thresholds[1]
                         "#ffbbbb"
@@ -215,12 +214,65 @@ function create_app()
 
     end
 
+    # upload
+
+    function upload_selection(tag)
+        reference_path = joinpath(root_path, "reference")
+
+        @info "Downloading latest reference image folder for $tag"
+        tmpdir = try
+            download_refimages(tag)
+        catch e
+            @error "Failed to download refimg folder. Is the tag $tag correct?" exception = (e, catch_backtrace())
+            return
+        end
+
+        @info "Updating files in $tmpdir"
+
+        for image in marked_for_upload
+            @info "Overwriting or adding $image"
+            target = joinpath(tmpdir, image)
+            # make sure the path exists
+            mkpath(splitdir(target)[1])
+
+            source = joinpath(reference_path, image)
+            cp(source, target, force = true)
+        end
+
+        for image in marked_for_deletion
+            @info "Deleting $image"
+            target = joinpath(tmpdir, image)
+            if isfile(target)
+                rm(target)
+            else
+                @warn "Cannot delete $image - does not exist."
+            end
+        end
+
+        try
+            upload_reference_images(tmpdir, tag)
+            @info "Upload successful."
+        catch e
+            @error "Upload failed: " exception = (e, catch_backtrace())
+        finally
+            @info "Deleting temp directory"
+            rm(tmpdir)
+            @info "You can ctrl+c out now."
+        end
+        return
+    end
+
+    # TODO: no less than 8rem width?
+    tag_textfield = Bonito.TextField("$(last_major_version())", style = Styles("width" => "8rem"))
+    upload_button = Bonito.Button("Update reference images with selection")
+    on(_ -> upload_selection(tag_textfield.value[]), upload_button.value)
+
     # Create page
 
     update_section = DOM.div(
         DOM.h2("Images to update"),
-        DOM.div("After pressing the button you will be asked which version to upload the reference images listed below to. After that the reference images on github will be replaced with an updated set if you have the rights to do so."),
-        Bonito.Button("Update reference images with selection"),
+        DOM.div("Pressing the button below will download the latest reference images for the selected version; add, update and/or remove the selected images listed below and then upload the changed reference image folder. See Julia terminal for progress updates."),
+        DOM.div(tag_textfield, upload_button),
         DOM.h3(map(set -> "$(length(set)) images selected for updating:", marked_for_upload)),
         map(set -> DOM.ul([DOM.li(name) for name in set]), marked_for_upload),
         DOM.h3(map(set -> "$(length(set)) images selected for removal:", marked_for_deletion)),
@@ -230,14 +282,14 @@ function create_app()
     new_image_section = DOM.div(
         DOM.h2("New images without references"),
         DOM.div("The selected CI run produced an image for which no reference image exists. Selected images will be added as new reference images."),
-        new_checkbox,
+        new_checkbox, DOM.br(),
         Grid(new_cards, columns = "1fr 1fr 1fr")
     )
 
     missing_recordings_section = DOM.div(
         DOM.h2("Old reference images without recordings"),
         DOM.div("The selected CI run did not produce an image, but a reference image exists. This implies that a reference test was deleted or renamed. Selected images will be deleted from the reference images."),
-        missing_checkbox,
+        missing_checkbox, DOM.br(),
         Grid(missing_cards, columns = "1fr 1fr 1fr")
     )
 
@@ -253,15 +305,16 @@ function create_app()
         new_image_section,
         missing_recordings_section,
         main_section,
-        style = Styles(CSS("font-family" => "sans-serif"))
+        style = Styles(CSS("font-family" => "sans-serif", "min-width" => "4rem"))
     )
 end
 
-
-
-if @isdefined server
-    close(server)
+function create_app(root_path)
+    return App(() -> create_app_content(root_path))
 end
-server = Bonito.Server("0.0.0.0", 8080)
-display(server)
-route!(server, "/" => App(create_app))
\ No newline at end of file
+
+function create_browser_display(root_path, ip = "0.0.0.0", port = 8080)
+    server = Bonito.Server(ip, port)
+    route!(server, "/" => create_app(root_path))
+    return server
+end
\ No newline at end of file

From 69dc091001fed14b914cde9a5797d01e7dbc24ac Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 15:15:14 +0100
Subject: [PATCH 18/26] close previous server before creating a new one

---
 ReferenceUpdater/src/bonito-app.jl | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index b924234cbd7..fb2ad60409e 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -313,8 +313,10 @@ function create_app(root_path)
     return App(() -> create_app_content(root_path))
 end
 
-function create_browser_display(root_path, ip = "0.0.0.0", port = 8080)
-    server = Bonito.Server(ip, port)
-    route!(server, "/" => create_app(root_path))
-    return server
+const server = Ref{Any}()
+function create_browser_display(root_path)
+    isassigned(server) && close(server)
+    server[] = Bonito.Server("0.0.0.0", 8080)
+    route!(server[], "/" => create_app(root_path))
+    return server[]
 end
\ No newline at end of file

From 8c94cfef60bd69a7c5d00a446f363952f596fae0 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 15:35:35 +0100
Subject: [PATCH 19/26] add commit/pr entrypoint, fix upload, some cleanup

---
 ReferenceUpdater/src/bonito-app.jl | 66 ++++++++++++++++++++----------
 1 file changed, 44 insertions(+), 22 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index fb2ad60409e..e2995432a9c 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -62,7 +62,6 @@ function create_app_content(root_path::String)
                 delete!(marked[], current_file)
             end
             notify(marked)
-            @info marked[]
         end
         return DOM.div(
             Checkbox(local_marked, Dict{Symbol, Any}(:style => checkbox_style)),
@@ -217,47 +216,58 @@ function create_app_content(root_path::String)
     # upload
 
     function upload_selection(tag)
-        reference_path = joinpath(root_path, "reference")
+        recorded_path = joinpath(root_path, "recorded")
 
         @info "Downloading latest reference image folder for $tag"
         tmpdir = try
             download_refimages(tag)
         catch e
-            @error "Failed to download refimg folder. Is the tag $tag correct?" exception = (e, catch_backtrace())
+            @error "Failed to download refimg folder. Is the tag $tag correct? Exiting without upload." exception = (e, catch_backtrace())
             return
         end
 
         @info "Updating files in $tmpdir"
 
-        for image in marked_for_upload
-            @info "Overwriting or adding $image"
-            target = joinpath(tmpdir, image)
-            # make sure the path exists
-            mkpath(splitdir(target)[1])
-
-            source = joinpath(reference_path, image)
-            cp(source, target, force = true)
+        try
+            for image in marked_for_upload[]
+                @info "Overwriting or adding $image"
+                target = joinpath(tmpdir, normpath(image))
+                # make sure the path exists
+                mkpath(splitdir(target)[1])
+
+                source = joinpath(recorded_path, normpath(image))
+                cp(source, target, force = true)
+            end
+        catch e
+            @error "Failed to overwrite/add images. Exiting without upload." exception = (e, catch_backtrace())
+            return
         end
 
-        for image in marked_for_deletion
-            @info "Deleting $image"
-            target = joinpath(tmpdir, image)
-            if isfile(target)
-                rm(target)
-            else
-                @warn "Cannot delete $image - does not exist."
+        try
+            for image in marked_for_deletion[]
+                @info "Deleting $image"
+                target = joinpath(tmpdir, normpath(image))
+                if isfile(target)
+                    rm(target)
+                else
+                    @warn "Cannot delete $image - does not exist."
+                end
             end
+        catch e
+            @error "Failed to remove images. Exiting without upload." exception = (e, catch_backtrace())
+            return
         end
 
         try
+            @info "Uploading..."
             upload_reference_images(tmpdir, tag)
             @info "Upload successful."
         catch e
             @error "Upload failed: " exception = (e, catch_backtrace())
         finally
-            @info "Deleting temp directory"
+            @info "Deleting temp directory..."
             rm(tmpdir)
-            @info "You can ctrl+c out now."
+            @info "Done. You can ctrl+c out now."
         end
         return
     end
@@ -309,14 +319,26 @@ function create_app_content(root_path::String)
     )
 end
 
-function create_app(root_path)
+function create_app_from_dir(root_path)
     return App(() -> create_app_content(root_path))
 end
 
 const server = Ref{Any}()
-function create_browser_display(root_path)
+function create_browser_display_from_dir(root_path)
     isassigned(server) && close(server)
     server[] = Bonito.Server("0.0.0.0", 8080)
     route!(server[], "/" => create_app(root_path))
     return server[]
+end
+
+function create_app(; commit = nothing, pr = nothing)
+    tmpdir = download_artifacts(commit = commit, pr = pr)
+    @info "Creating Bonito app from folder $tmpdir."
+    return create_app_from_dir(tmpdir)
+end
+
+function create_browser_display(; commit = nothing, pr = nothing)
+    tmpdir = download_artifacts(commit = commit, pr = pr)
+    @info "Creating Bonito app from folder $tmpdir."
+    return create_browser_display_from_dir(tmpdir)
 end
\ No newline at end of file

From a67d9d21b3b7178a6cc3e07cd0d8087ca30b2954 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 16:47:11 +0100
Subject: [PATCH 20/26] try generating GLMakie diff scores

---
 .github/workflows/reference_tests.yml      | 21 +++++++++++++
 ReferenceTests/src/ReferenceTests.jl       |  1 +
 ReferenceTests/src/cross_backend_scores.jl | 35 ++++++++++++++++++++++
 ReferenceTests/src/runtests.jl             |  7 +++--
 4 files changed, 61 insertions(+), 3 deletions(-)
 create mode 100644 ReferenceTests/src/cross_backend_scores.jl

diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml
index 3903929641f..c15dc47b249 100644
--- a/.github/workflows/reference_tests.yml
+++ b/.github/workflows/reference_tests.yml
@@ -187,6 +187,26 @@ jobs:
         with:
           name: ReferenceImages_GLMakie_1
           path: ./ReferenceImages/GLMakie
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+          arch: x64
+          os: 'ubuntu-20.04'
+      - uses: julia-actions/cache@v2
+      - name: Install Julia dependencies
+        shell: julia --project=monorepo {0}
+        run: |
+          using Pkg;
+          # dev mono repo versions
+          pkg"registry up"
+          Pkg.update()
+          pkg"dev . ./ReferenceTests"
+      - name: Generate GLMakie diff scores
+        continue-on-error: true
+        run: >
+          julia --color=yes --project=monorepo -e 'using ReferenceTests; ReferenceTests.generate_backend_comparison_scores("./ReferenceImages/")'
       - name: Consolidate reference image folders
         run: |
           baseDir="./ReferenceImages"
@@ -207,6 +227,7 @@ jobs:
               cat "${baseDir}/${dir}/scores.tsv" >> "./ReferenceImagesCombined/scores.tsv"
               cat "${baseDir}/${dir}/new_files.txt" >> "./ReferenceImagesCombined/new_files.txt"
               cat "${baseDir}/${dir}/missing_files.txt" >> "./ReferenceImagesCombined/missing_files.txt"
+              cat "${baseDir}/${dir}/cross_backend_scores.txt" >> "./ReferenceImagesCombined/cross_backend_scores.txt"
 
               # Copy recorded folder
               mkdir -p "./ReferenceImagesCombined/recorded/${dir}/"
diff --git a/ReferenceTests/src/ReferenceTests.jl b/ReferenceTests/src/ReferenceTests.jl
index 11189a69cd0..e5ce38f4739 100644
--- a/ReferenceTests/src/ReferenceTests.jl
+++ b/ReferenceTests/src/ReferenceTests.jl
@@ -40,6 +40,7 @@ include("database.jl")
 include("stable_rng.jl")
 include("runtests.jl")
 include("image_download.jl")
+include("cross_backend_scores.jl")
 
 export @include_reference_tests
 
diff --git a/ReferenceTests/src/cross_backend_scores.jl b/ReferenceTests/src/cross_backend_scores.jl
new file mode 100644
index 00000000000..52417b23338
--- /dev/null
+++ b/ReferenceTests/src/cross_backend_scores.jl
@@ -0,0 +1,35 @@
+function generate_backend_comparison_scores(root_folder)
+    ref = joinpath(root_folder, "GLMakie", "recorded", "GLMakie")
+    isdir(ref) || error("GLMakie subfolder $ref must exist")
+
+    for folder in readdir(root_folder)
+        isdir(joinpath(root_folder, folder)) || continue
+
+        if folder == "GLMakie" # create dummy file
+            close(open(joinpath(root_folder, folder, "cross_backend_scores.tsv"), "w"))
+            continue
+        end
+
+        @info "Generating comparison scores between $folder and GLMakie"
+        generate_backend_comparison_scores(joinpath(root_folder, folder, "recorded", folder), ref)
+    end
+
+    return
+end
+
+function generate_backend_comparison_scores(target_dir, reference_dir)
+    isdir(target_dir) || error("Invalid directory: $target_dir")
+    isdir(reference_dir) || error("Invalid directory: $reference_dir")
+
+    target_files = get_all_relative_filepaths_recursively(target_dir)
+
+    open(joinpath(target_dir, "../../cross_backend_scores.tsv"), "w") do file
+        for filepath in target_files
+            isfile(joinpath(reference_dir, filepath)) || continue
+            diff = compare_media(joinpath(target_dir, filepath),
+                                    joinpath(reference_dir, filepath))
+            println(file, diff, '\t', filepath)
+        end
+    end
+end
+
diff --git a/ReferenceTests/src/runtests.jl b/ReferenceTests/src/runtests.jl
index 52a689289d0..b6f107dd53e 100644
--- a/ReferenceTests/src/runtests.jl
+++ b/ReferenceTests/src/runtests.jl
@@ -42,6 +42,7 @@ function compare_images(a::AbstractMatrix{<:Union{RGB,RGBA}}, b::AbstractMatrix{
 
     _norm(rgb1::RGBf, rgb2::RGBf) = sqrt(sum(((rgb1.r - rgb2.r)^2, (rgb1.g - rgb2.g)^2, (rgb1.b - rgb2.b)^2)))
     _norm(rgba1::RGBAf, rgba2::RGBAf) = sqrt(sum(((rgba1.r - rgba2.r)^2, (rgba1.g - rgba2.g)^2, (rgba1.b - rgba2.b)^2, (rgba1.alpha - rgba2.alpha)^2)))
+    _norm(c1::Colorant, c2::Colorant) = _norm(RGBAf(c1), RGBAf(c2))
 
     # compute the difference score as the maximum of the mean squared differences over the color
     # values of tiles over the image. using tiles is a simple way to increase the local sensitivity
@@ -99,7 +100,7 @@ function record_comparison(base_folder::String, backend::String; record_folder_n
             println(file, path)
         end
     end
-    
+
     open(joinpath(base_folder, "missing_files.txt"), "w") do file
         backend_ref_dir = joinpath(reference_folder, backend)
         recorded_paths = mapreduce(vcat, walkdir(backend_ref_dir)) do (root, dirs, files)
@@ -134,8 +135,8 @@ function test_comparison(scores; threshold)
 end
 
 function compare(
-        relative_test_paths::Vector{String}, reference_dir::String, record_dir; 
-        o_refdir = reference_dir, missing_refimages = String[], 
+        relative_test_paths::Vector{String}, reference_dir::String, record_dir;
+        o_refdir = reference_dir, missing_refimages = String[],
         scores = Dict{String,Float64}()
     )
 

From 8e8ad5e25b57d61e3a406286cda556772d9151ea Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 22:07:25 +0100
Subject: [PATCH 21/26] revert ci script

---
 .github/workflows/reference_tests.yml | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/.github/workflows/reference_tests.yml b/.github/workflows/reference_tests.yml
index c15dc47b249..2c9391bbec1 100644
--- a/.github/workflows/reference_tests.yml
+++ b/.github/workflows/reference_tests.yml
@@ -187,26 +187,6 @@ jobs:
         with:
           name: ReferenceImages_GLMakie_1
           path: ./ReferenceImages/GLMakie
-      - name: Checkout
-        uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v2
-        with:
-          version: '1'
-          arch: x64
-          os: 'ubuntu-20.04'
-      - uses: julia-actions/cache@v2
-      - name: Install Julia dependencies
-        shell: julia --project=monorepo {0}
-        run: |
-          using Pkg;
-          # dev mono repo versions
-          pkg"registry up"
-          Pkg.update()
-          pkg"dev . ./ReferenceTests"
-      - name: Generate GLMakie diff scores
-        continue-on-error: true
-        run: >
-          julia --color=yes --project=monorepo -e 'using ReferenceTests; ReferenceTests.generate_backend_comparison_scores("./ReferenceImages/")'
       - name: Consolidate reference image folders
         run: |
           baseDir="./ReferenceImages"
@@ -227,7 +207,6 @@ jobs:
               cat "${baseDir}/${dir}/scores.tsv" >> "./ReferenceImagesCombined/scores.tsv"
               cat "${baseDir}/${dir}/new_files.txt" >> "./ReferenceImagesCombined/new_files.txt"
               cat "${baseDir}/${dir}/missing_files.txt" >> "./ReferenceImagesCombined/missing_files.txt"
-              cat "${baseDir}/${dir}/cross_backend_scores.txt" >> "./ReferenceImagesCombined/cross_backend_scores.txt"
 
               # Copy recorded folder
               mkdir -p "./ReferenceImagesCombined/recorded/${dir}/"
@@ -251,4 +230,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: n_missing_refimages
-          path: n_missing/
+          path: n_missing/
\ No newline at end of file

From c2dd48a559c4aae6c6ed5da85fb2b928a659cc0b Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 22:08:34 +0100
Subject: [PATCH 22/26] handle GLMakie compare in ReferenceUpdater

---
 ReferenceTests/src/ReferenceTests.jl        |  1 +
 ReferenceTests/src/compare_media.jl         | 89 ++++++++++++++++++++
 ReferenceTests/src/runtests.jl              | 84 -------------------
 ReferenceUpdater/Project.toml               |  8 ++
 ReferenceUpdater/src/bonito-app.jl          | 91 ++++++++++++++++++++-
 ReferenceUpdater/src/cross_backend_score.jl | 24 ++++++
 6 files changed, 209 insertions(+), 88 deletions(-)
 create mode 100644 ReferenceTests/src/compare_media.jl
 create mode 100644 ReferenceUpdater/src/cross_backend_score.jl

diff --git a/ReferenceTests/src/ReferenceTests.jl b/ReferenceTests/src/ReferenceTests.jl
index e5ce38f4739..1ce154d2182 100644
--- a/ReferenceTests/src/ReferenceTests.jl
+++ b/ReferenceTests/src/ReferenceTests.jl
@@ -38,6 +38,7 @@ using Images, FixedPointNumbers, Colors, ColorTypes
 
 include("database.jl")
 include("stable_rng.jl")
+include("compare_media.jl")
 include("runtests.jl")
 include("image_download.jl")
 include("cross_backend_scores.jl")
diff --git a/ReferenceTests/src/compare_media.jl b/ReferenceTests/src/compare_media.jl
new file mode 100644
index 00000000000..11c6ef614a1
--- /dev/null
+++ b/ReferenceTests/src/compare_media.jl
@@ -0,0 +1,89 @@
+function get_frames(a, b)
+    return (get_frames(a), get_frames(b))
+end
+
+rgbf_convert(x::AbstractMatrix{<:RGB}) = convert(Matrix{RGBf}, x)
+rgbf_convert(x::AbstractMatrix{<:RGBA}) = convert(Matrix{RGBAf}, x)
+
+# pulled from Makie so we don't need to include it
+function extract_frames(video, frame_folder; loglevel="quiet")
+    path = joinpath(frame_folder, "frame%04d.png")
+    run(`$(FFMPEG_jll.ffmpeg()) -loglevel $(loglevel) -i $video -y $path`)
+end
+
+function get_frames(video::AbstractString)
+    mktempdir() do folder
+        afolder = joinpath(folder, "a")
+        mkpath(afolder)
+        extract_frames(video, afolder)
+        aframes = joinpath.(afolder, readdir(afolder))
+        if length(aframes) > 10
+            # we don't want to compare too many frames since it's time costly
+            # so we just compare 10 random frames if more than 10
+            samples = range(1, stop=length(aframes), length=10)
+            istep = round(Int, length(aframes) / 10)
+            samples = 1:istep:length(aframes)
+            aframes = aframes[samples]
+        end
+        return load.(aframes)
+    end
+end
+
+function compare_images(a::AbstractMatrix{<:Union{RGB,RGBA}}, b::AbstractMatrix{<:Union{RGB,RGBA}})
+
+    a = rgbf_convert(a)
+    b = rgbf_convert(b)
+
+    if size(a) != size(b)
+        @warn "images don't have the same size, difference will be Inf"
+        return Inf
+    end
+
+    approx_tile_size_px = 30
+
+    range_dim1 = round.(Int, range(0, size(a, 1), length = ceil(Int, size(a, 1) / approx_tile_size_px)))
+    range_dim2 = round.(Int, range(0, size(a, 2), length = ceil(Int, size(a, 2) / approx_tile_size_px)))
+
+    boundary_iter(boundaries) = zip(boundaries[1:end-1] .+ 1, boundaries[2:end])
+
+    _norm(rgb1::RGBf, rgb2::RGBf) = sqrt(sum(((rgb1.r - rgb2.r)^2, (rgb1.g - rgb2.g)^2, (rgb1.b - rgb2.b)^2)))
+    _norm(rgba1::RGBAf, rgba2::RGBAf) = sqrt(sum(((rgba1.r - rgba2.r)^2, (rgba1.g - rgba2.g)^2, (rgba1.b - rgba2.b)^2, (rgba1.alpha - rgba2.alpha)^2)))
+    _norm(c1::Colorant, c2::Colorant) = _norm(RGBAf(c1), RGBAf(c2))
+
+    # compute the difference score as the maximum of the mean squared differences over the color
+    # values of tiles over the image. using tiles is a simple way to increase the local sensitivity
+    # without directly going to pixel-based comparison
+    # it also makes the scores more comparable between reference images of different sizes, because the same
+    # local differences would be normed to different mean scores if the images have different numbers of pixels
+    return maximum(Iterators.product(boundary_iter(range_dim1), boundary_iter(range_dim2))) do ((mi1, ma1), (mi2, ma2))
+        @views mean(_norm.(a[mi1:ma1, mi2:ma2], b[mi1:ma1, mi2:ma2]))
+    end
+end
+
+function compare_media(a::AbstractString, b::AbstractString)
+    _, ext = splitext(a)
+    if ext in (".png", ".jpg", ".jpeg", ".JPEG", ".JPG")
+        imga = load(a)
+        imgb = load(b)
+        return compare_images(imga, imgb)
+    elseif ext in (".mp4", ".gif")
+        aframes = get_frames(a)
+        bframes = get_frames(b)
+        # Frames can differ in length, which usually shouldn't be the case but can happen
+        # when the implementation of record changes, or when the example changes its number of frames
+        # In that case, we just return inf + warn
+        if length(aframes) != length(bframes)
+            @warn "not the same number of frames in video, difference will be Inf"
+            return Inf
+        end
+        return maximum(compare_images.(aframes, bframes))
+    else
+        error("Unknown media extension: $ext")
+    end
+end
+
+function get_all_relative_filepaths_recursively(dir)
+    mapreduce(vcat, walkdir(dir)) do (root, dirs, files)
+        relpath.(joinpath.(root, files), dir)
+    end
+end
\ No newline at end of file
diff --git a/ReferenceTests/src/runtests.jl b/ReferenceTests/src/runtests.jl
index b6f107dd53e..7f74c5cf5fa 100644
--- a/ReferenceTests/src/runtests.jl
+++ b/ReferenceTests/src/runtests.jl
@@ -1,87 +1,3 @@
-function get_frames(a, b)
-    return (get_frames(a), get_frames(b))
-end
-
-rgbf_convert(x::AbstractMatrix{<:RGB}) = convert(Matrix{RGBf}, x)
-rgbf_convert(x::AbstractMatrix{<:RGBA}) = convert(Matrix{RGBAf}, x)
-
-function get_frames(video::AbstractString)
-    mktempdir() do folder
-        afolder = joinpath(folder, "a")
-        mkpath(afolder)
-        Makie.extract_frames(video, afolder)
-        aframes = joinpath.(afolder, readdir(afolder))
-        if length(aframes) > 10
-            # we don't want to compare too many frames since it's time costly
-            # so we just compare 10 random frames if more than 10
-            samples = range(1, stop=length(aframes), length=10)
-            istep = round(Int, length(aframes) / 10)
-            samples = 1:istep:length(aframes)
-            aframes = aframes[samples]
-        end
-        return load.(aframes)
-    end
-end
-
-function compare_images(a::AbstractMatrix{<:Union{RGB,RGBA}}, b::AbstractMatrix{<:Union{RGB,RGBA}})
-
-    a = rgbf_convert(a)
-    b = rgbf_convert(b)
-
-    if size(a) != size(b)
-        @warn "images don't have the same size, difference will be Inf"
-        return Inf
-    end
-
-    approx_tile_size_px = 30
-
-    range_dim1 = round.(Int, range(0, size(a, 1), length = ceil(Int, size(a, 1) / approx_tile_size_px)))
-    range_dim2 = round.(Int, range(0, size(a, 2), length = ceil(Int, size(a, 2) / approx_tile_size_px)))
-
-    boundary_iter(boundaries) = zip(boundaries[1:end-1] .+ 1, boundaries[2:end])
-
-    _norm(rgb1::RGBf, rgb2::RGBf) = sqrt(sum(((rgb1.r - rgb2.r)^2, (rgb1.g - rgb2.g)^2, (rgb1.b - rgb2.b)^2)))
-    _norm(rgba1::RGBAf, rgba2::RGBAf) = sqrt(sum(((rgba1.r - rgba2.r)^2, (rgba1.g - rgba2.g)^2, (rgba1.b - rgba2.b)^2, (rgba1.alpha - rgba2.alpha)^2)))
-    _norm(c1::Colorant, c2::Colorant) = _norm(RGBAf(c1), RGBAf(c2))
-
-    # compute the difference score as the maximum of the mean squared differences over the color
-    # values of tiles over the image. using tiles is a simple way to increase the local sensitivity
-    # without directly going to pixel-based comparison
-    # it also makes the scores more comparable between reference images of different sizes, because the same
-    # local differences would be normed to different mean scores if the images have different numbers of pixels
-    return maximum(Iterators.product(boundary_iter(range_dim1), boundary_iter(range_dim2))) do ((mi1, ma1), (mi2, ma2))
-        @views mean(_norm.(a[mi1:ma1, mi2:ma2], b[mi1:ma1, mi2:ma2]))
-    end
-end
-
-function compare_media(a::AbstractString, b::AbstractString)
-    _, ext = splitext(a)
-    if ext in (".png", ".jpg", ".jpeg", ".JPEG", ".JPG")
-        imga = load(a)
-        imgb = load(b)
-        return compare_images(imga, imgb)
-    elseif ext in (".mp4", ".gif")
-        aframes = get_frames(a)
-        bframes = get_frames(b)
-        # Frames can differ in length, which usually shouldn't be the case but can happen
-        # when the implementation of record changes, or when the example changes its number of frames
-        # In that case, we just return inf + warn
-        if length(aframes) != length(bframes)
-            @warn "not the same number of frames in video, difference will be Inf"
-            return Inf
-        end
-        return maximum(compare_images.(aframes, bframes))
-    else
-        error("Unknown media extension: $ext")
-    end
-end
-
-function get_all_relative_filepaths_recursively(dir)
-    mapreduce(vcat, walkdir(dir)) do (root, dirs, files)
-        relpath.(joinpath.(root, files), dir)
-    end
-end
-
 function record_comparison(base_folder::String, backend::String; record_folder_name="recorded", tag=last_major_version())
     record_folder = joinpath(base_folder, record_folder_name)
     @info "Downloading reference images"
diff --git a/ReferenceUpdater/Project.toml b/ReferenceUpdater/Project.toml
index 9578e87af8f..10dc8bf1611 100644
--- a/ReferenceUpdater/Project.toml
+++ b/ReferenceUpdater/Project.toml
@@ -5,12 +5,16 @@ version = "0.1.0"
 
 [deps]
 Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8"
+Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+FFMPEG_jll = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5"
+FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
@@ -18,5 +22,9 @@ ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]
 Bonito = "3, 4"
+Colors = "0.13.0"
 DelimitedFiles = "1.9.1"
+FFMPEG_jll = "6.1.2"
+FileIO = "1.16.6"
 HTTP = "1"
+Statistics = "1.11.1"
diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index e2995432a9c..2f106c616b5 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -1,5 +1,6 @@
 using Bonito, DelimitedFiles
 
+include("cross_backend_score.jl")
 
 """
     create_app(path_to_reference_images_folder)
@@ -52,6 +53,8 @@ function create_app_content(root_path::String)
     function refimage_selection_checkbox(marked, current_file, mark_all = nothing)
         if mark_all === nothing
             local_marked = Observable(false)
+        elseif mark_all isa Bool
+            local_marked = Observable(mark_all)
         else
             local_marked = map(identity, mark_all)
         end
@@ -259,14 +262,14 @@ function create_app_content(root_path::String)
         end
 
         try
-            @info "Uploading..."
-            upload_reference_images(tmpdir, tag)
+            @info "TODO: Uploading..."
+            # upload_reference_images(tmpdir, tag)
             @info "Upload successful."
         catch e
             @error "Upload failed: " exception = (e, catch_backtrace())
         finally
-            @info "Deleting temp directory..."
-            rm(tmpdir)
+            @info "TODO: Deleting temp directory..."
+            # rm(tmpdir)
             @info "Done. You can ctrl+c out now."
         end
         return
@@ -277,6 +280,78 @@ function create_app_content(root_path::String)
     upload_button = Bonito.Button("Update reference images with selection")
     on(_ -> upload_selection(tag_textfield.value[]), upload_button.value)
 
+    # compare to GLMakie
+
+    compare_button = Bonito.Button("Compare current selection")
+    glmakie_compare_grid = Observable{Any}(DOM.div())
+    on(compare_button.value) do _
+
+        without_backend = unique(map(collect(marked_for_upload[])) do img
+            replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
+        end)
+        sort!(without_backend; by=get_score, rev=true)
+
+        file2score = compare_selection(root_path, marked_for_upload[])
+
+        updated_cards = Any[]
+        for img_name in without_backend
+            for backend in backends
+                backend == "GLMakie" && continue
+
+                current_file = backend * "/" * img_name
+                if haskey(file2score, current_file)
+                    cb = refimage_selection_checkbox(marked_for_upload, current_file, true)
+
+                    score = round(get(file2score, current_file, -1.0); digits=3)
+                    # TODO: Is there a better way to handle default with overwrites?
+                    card_style = Styles(card_css, CSS(
+                        "background-color" => if score > score_thresholds[1]
+                            "#ffbbbb"
+                        elseif score > score_thresholds[2]
+                            "#ffddbb"
+                        elseif score > score_thresholds[3]
+                            "#ffffdd"
+                        elseif score < -0.1
+                            "#bbbbff"
+                        else
+                            "#eeeeee"
+                        end
+                    ))
+
+                    path_button = Bonito.Button("", style = button_style)
+                    selection = 2 # Recorded (new), Reference (old)
+                    ref_name = "GLMakie/$img_name"
+                    local_path = map(path_button.value) do click
+                        selection = mod1(selection + 1, 2)
+                        path_button.content[] = [backend, "GLMakie"][selection]
+                        file = [current_file, ref_name][selection]
+                        local_path = normpath(joinpath(root_path, "recorded", file))
+                        return Bonito.Asset(local_path)
+                    end
+
+                    media = media_element(img_name, local_path)
+
+                    card = Card(
+                        DOM.div(
+                            cb,
+                            DOM.div(
+                                path_button,
+                                DOM.div("Score: $score", style = score_style)
+                            ),
+                            media
+                        ),
+                        style = card_style
+                    )
+                    push!(updated_cards, card)
+                else
+                    push!(updated_cards, DOM.div())
+                end
+            end
+        end
+
+        glmakie_compare_grid[] = Grid(updated_cards, columns = "33% 33%")
+    end
+
     # Create page
 
     update_section = DOM.div(
@@ -289,6 +364,13 @@ function create_app_content(root_path::String)
         map(set -> DOM.ul([DOM.li(name) for name in set]), marked_for_deletion),
     )
 
+    glmakie_compare_section = DOM.div(
+        DOM.h2("Compare Selection to GLMakie"),
+        DOM.div("Compares every selected refimg with it's corresponding GLMakie refimg. If there is none the score will be -1.0."),
+        compare_button,
+        glmakie_compare_grid
+    )
+
     new_image_section = DOM.div(
         DOM.h2("New images without references"),
         DOM.div("The selected CI run produced an image for which no reference image exists. Selected images will be added as new reference images."),
@@ -312,6 +394,7 @@ function create_app_content(root_path::String)
 
     return DOM.div(
         update_section,
+        glmakie_compare_section,
         new_image_section,
         missing_recordings_section,
         main_section,
diff --git a/ReferenceUpdater/src/cross_backend_score.jl b/ReferenceUpdater/src/cross_backend_score.jl
new file mode 100644
index 00000000000..d4fcf748d65
--- /dev/null
+++ b/ReferenceUpdater/src/cross_backend_score.jl
@@ -0,0 +1,24 @@
+# Avoid loading all of ReferenceTests with its heavy dependencies
+using Colors, FileIO, Statistics
+import FFMPEG_jll
+
+const RGBf = RGB{Float32}
+const RGBAf = RGBA{Float32}
+
+include("../../ReferenceTests/src/compare_media.jl")
+include("../../ReferenceTests/src/cross_backend_scores.jl")
+
+function compare_selection(root_path, files)
+    file2score = Dict{String, Float64}()
+    for filename in files
+        to_check = joinpath(root_path, "recorded", normpath(filename))
+        ref_file = replace(filename, r"(GLMakie|CairoMakie|WGLMakie)/" => "GLMakie/")
+        reference = joinpath(root_path, "recorded", normpath(ref_file))
+        if isfile(to_check) && isfile(reference)
+            file2score[filename] = compare_media(to_check, reference)
+        else
+            file2score[filename] = -1.0
+        end
+    end
+    return file2score
+end
\ No newline at end of file

From 162f432e3f5ac1f06ed2ce42f56c365afe37a635 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 22:47:40 +0100
Subject: [PATCH 23/26] fix import

---
 ReferenceTests/src/compare_media.jl | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/ReferenceTests/src/compare_media.jl b/ReferenceTests/src/compare_media.jl
index 11c6ef614a1..06d59631f4c 100644
--- a/ReferenceTests/src/compare_media.jl
+++ b/ReferenceTests/src/compare_media.jl
@@ -5,10 +5,14 @@ end
 rgbf_convert(x::AbstractMatrix{<:RGB}) = convert(Matrix{RGBf}, x)
 rgbf_convert(x::AbstractMatrix{<:RGBA}) = convert(Matrix{RGBAf}, x)
 
-# pulled from Makie so we don't need to include it
-function extract_frames(video, frame_folder; loglevel="quiet")
-    path = joinpath(frame_folder, "frame%04d.png")
-    run(`$(FFMPEG_jll.ffmpeg()) -loglevel $(loglevel) -i $video -y $path`)
+if @isdefined Makie
+    using Makie: extract_frames
+else
+    # pulled from Makie so we don't need to include it
+    function extract_frames(video, frame_folder; loglevel="quiet")
+        path = joinpath(frame_folder, "frame%04d.png")
+        run(`$(FFMPEG_jll.ffmpeg()) -loglevel $(loglevel) -i $video -y $path`)
+    end
 end
 
 function get_frames(video::AbstractString)

From 292854a10d59cc3e90412374ea75509694c0a682 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Thu, 23 Jan 2025 23:09:29 +0100
Subject: [PATCH 24/26] add toggle-all-above-score checkbox

---
 ReferenceUpdater/src/bonito-app.jl | 45 +++++++++++++++++++++++++-----
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 2f106c616b5..87465400025 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -20,7 +20,7 @@ function create_app_content(root_path::String)
     )
 
     score_style = Styles(
-        CSS("font-size" => "10", "font-weight" => "normal"),
+        CSS("font-size" => "12", "font-weight" => "normal"),
         CSS("width" => "fit-content",
             "padding-right" => "6px", "padding-left" => "6px",
             "padding-bottom" => "2px", "padding-top" => "2px",
@@ -31,6 +31,8 @@ function create_app_content(root_path::String)
 
     checkbox_style = Styles(CSS("transform" => "scale(1)"))
 
+    textfield_style = Styles(CSS("width" => "4rem", "font-size" => "12", "font-weight" => "normal"))
+
     card_css = CSS(
         "margin" => "0.1em", # outside
         "padding" => "0.5em", # inside
@@ -162,6 +164,14 @@ function create_app_content(root_path::String)
 
     sort!(imgs_with_score; by=get_score, rev=true)
 
+    update_multi_check = Observable(false)
+    update_multi_textinput = Bonito.TextField("0.05", style = textfield_style)
+    update_multi_checkbox  = DOM.div(
+        Checkbox(update_multi_check, Dict{Symbol, Any}(:style => checkbox_style)),
+        " Toggle All with Score ≥",
+        update_multi_textinput
+    )
+
     updated_cards = Any[]
     for img_name in imgs_with_score
         for backend in backends
@@ -169,9 +179,29 @@ function create_app_content(root_path::String)
             current_file = backend * "/" * img_name
             if haskey(lookup, current_file)
 
-                cb = refimage_selection_checkbox(marked_for_upload, current_file)
-
                 score = round(lookup[current_file]; digits=3)
+
+                local_marked = map(update_multi_check) do active
+                    try
+                        threshold = parse(Float64, update_multi_textinput.value[])
+                        return active && (score >= threshold)
+                    catch e
+                        return false
+                    end
+                end
+                on(local_marked) do is_marked
+                    if is_marked
+                        push!(marked_for_upload[], current_file)
+                    else
+                        delete!(marked_for_upload[], current_file)
+                    end
+                    notify(marked_for_upload)
+                end
+                cb = DOM.div(
+                    Checkbox(local_marked, Dict{Symbol, Any}(:style => checkbox_style)),
+                    " $current_file"
+                )
+
                 # TODO: Is there a better way to handle default with overwrites?
                 card_style = Styles(card_css, CSS(
                     "background-color" => if score > score_thresholds[1]
@@ -262,14 +292,14 @@ function create_app_content(root_path::String)
         end
 
         try
-            @info "TODO: Uploading..."
-            # upload_reference_images(tmpdir, tag)
+            @info "Uploading..."
+            upload_reference_images(tmpdir, tag)
             @info "Upload successful."
         catch e
             @error "Upload failed: " exception = (e, catch_backtrace())
         finally
-            @info "TODO: Deleting temp directory..."
-            # rm(tmpdir)
+            @info "Deleting temp directory..."
+            rm(tmpdir)
             @info "Done. You can ctrl+c out now."
         end
         return
@@ -388,6 +418,7 @@ function create_app_content(root_path::String)
     main_section = DOM.div(
         DOM.h2("Images with references"),
         DOM.div("This is the normal case where the selected CI run produced an image and the reference image exists. Each row shows one image per backend from the same reference image test, which can be compared with its reference image. Rows are sorted based on the maximum row score (bigger = more different). Background colors are based on this score, with red > $(score_thresholds[1]), orange > $(score_thresholds[2]), yellow > $(score_thresholds[3]) and the rest being light gray."),
+        update_multi_checkbox,
         DOM.br(),
         Grid(updated_cards, columns = "1fr 1fr 1fr")
     )

From 58beb3122861511ecce45f608b9e36c248d2e7a5 Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Fri, 24 Jan 2025 14:01:06 +0100
Subject: [PATCH 25/26] connect selection checkboxes & fix backend order

---
 ReferenceTests/src/compare_media.jl |  2 ++
 ReferenceUpdater/src/bonito-app.jl  | 21 ++++++++++-----------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/ReferenceTests/src/compare_media.jl b/ReferenceTests/src/compare_media.jl
index 06d59631f4c..9ed4c054f1a 100644
--- a/ReferenceTests/src/compare_media.jl
+++ b/ReferenceTests/src/compare_media.jl
@@ -1,3 +1,5 @@
+# NOTE: This file is reused by ReferenceUpdater
+
 function get_frames(a, b)
     return (get_frames(a), get_frames(b))
 end
diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index 87465400025..a560de1babb 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -52,14 +52,11 @@ function create_app_content(root_path::String)
     selection_string = ["Showing new recorded", "Showing old reference"]
     score_thresholds = [0.05, 0.03, 0.01]
 
-    function refimage_selection_checkbox(marked, current_file, mark_all = nothing)
-        if mark_all === nothing
-            local_marked = Observable(false)
-        elseif mark_all isa Bool
-            local_marked = Observable(mark_all)
-        else
-            local_marked = map(identity, mark_all)
-        end
+    # "globals"
+    checkbox_obs = Dict{String, Observable{Bool}}()
+
+    function refimage_selection_checkbox(marked, current_file, obs = Observable(false))
+        local_marked = convert(Observable{Bool}, obs)
         on(local_marked) do is_marked
             if is_marked
                 push!(marked[], current_file)
@@ -110,7 +107,8 @@ function create_app_content(root_path::String)
 
                 current_file = backend * "/" * img_name
                 if current_file in files
-                    cb = refimage_selection_checkbox(marked, current_file, mark_all)
+                    checkbox_obs[current_file] = obs = map(identity, mark_all)
+                    cb = refimage_selection_checkbox(marked, current_file, obs)
                     local_path = Bonito.Asset(normpath(joinpath(root_path, image_folder, backend, img_name)))
                     media = media_element(img_name, local_path)
                     card = Card(DOM.div(cb, media), style = Styles(card_css, CSS("background-color" => "#eeeeee")))
@@ -150,7 +148,6 @@ function create_app_content(root_path::String)
     end)
 
     function get_score(img_name)
-        backends = ["CairoMakie", "GLMakie", "WGLMakie"]
         scores = map(backends) do backend
             name = backend * "/" * img_name
             if haskey(lookup, name)
@@ -201,6 +198,7 @@ function create_app_content(root_path::String)
                     Checkbox(local_marked, Dict{Symbol, Any}(:style => checkbox_style)),
                     " $current_file"
                 )
+                checkbox_obs[current_file] = local_marked
 
                 # TODO: Is there a better way to handle default with overwrites?
                 card_style = Styles(card_css, CSS(
@@ -330,7 +328,8 @@ function create_app_content(root_path::String)
 
                 current_file = backend * "/" * img_name
                 if haskey(file2score, current_file)
-                    cb = refimage_selection_checkbox(marked_for_upload, current_file, true)
+                    obs = checkbox_obs[current_file]
+                    cb = refimage_selection_checkbox(marked_for_upload, current_file, obs)
 
                     score = round(get(file2score, current_file, -1.0); digits=3)
                     # TODO: Is there a better way to handle default with overwrites?

From 6f1b2c3beea8e600c744e228c311c3cd691a9fde Mon Sep 17 00:00:00 2001
From: ffreyer <frederic481994@hotmail.de>
Date: Fri, 24 Jan 2025 14:11:15 +0100
Subject: [PATCH 26/26] fix sorting in GLMakie compare

---
 ReferenceUpdater/src/bonito-app.jl | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/ReferenceUpdater/src/bonito-app.jl b/ReferenceUpdater/src/bonito-app.jl
index a560de1babb..c002cf6ddc3 100644
--- a/ReferenceUpdater/src/bonito-app.jl
+++ b/ReferenceUpdater/src/bonito-app.jl
@@ -317,10 +317,21 @@ function create_app_content(root_path::String)
         without_backend = unique(map(collect(marked_for_upload[])) do img
             replace(img, r"(GLMakie|CairoMakie|WGLMakie)/" => "")
         end)
-        sort!(without_backend; by=get_score, rev=true)
-
         file2score = compare_selection(root_path, marked_for_upload[])
 
+        function get_score(img_name)
+            scores = map(backends) do backend
+                name = backend * "/" * img_name
+                if haskey(file2score, name)
+                    return file2score[name]
+                else
+                    return -Inf
+                end
+            end
+            return maximum(scores)
+        end
+        sort!(without_backend; by=get_score, rev=true)
+
         updated_cards = Any[]
         for img_name in without_backend
             for backend in backends
@@ -395,7 +406,7 @@ function create_app_content(root_path::String)
 
     glmakie_compare_section = DOM.div(
         DOM.h2("Compare Selection to GLMakie"),
-        DOM.div("Compares every selected refimg with it's corresponding GLMakie refimg. If there is none the score will be -1.0."),
+        DOM.div("Compares every selected refimg with it's corresponding GLMakie refimg. If there is none the score will be -1.0. (This may take a minute if all images are selected.)"),
         compare_button,
         glmakie_compare_grid
     )