From 54182dec74568b2d869143ce1eb49f926a3fe0df Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 16:11:03 -0700
Subject: [PATCH 01/52] refactor to use f-strings

---
 tests/glvis_driver.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 34556929..af3fb2d2 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -72,31 +72,31 @@ def compare_images(baseline_file, output_file, expect_fail=False):
             print("[FAIL] Differences were not detected in the control case.")
         else:
             print("[PASS] Images match.")
-    print("       actual ssim = {}, cutoff = {}".format(ssim, cutoff_ssim))
+    print(f"       actual ssim = {ssim}, cutoff = {cutoff_ssim}")
     return ssim >= cutoff_ssim if not expect_fail else ssim < cutoff_ssim
 
 # Function to test a given glvis command with a variety of key-based commands.
 # Not currently in use.
 def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
-    print("Testing {0}:{1}...".format(t_group, t_name))
+    print(f"Testing {t_group0}:{t_name1}..."))
     full_screenshot_cmd = cmd + screenshot_keys
-    cmd = "{0} {1} -k \"{2}\"".format(exec_path, exec_args, full_screenshot_cmd)
-    print("Exec: {}".format(cmd))
+    cmd = f"{exec_path0} {exec_args} -k \"{full_screenshot_cmd2}\""
+    print(f"Exec: {cmd}")
     ret = os.system(cmd + " > /dev/null 2>&1")
     if ret != 0:
-        print("[FAIL] GLVis exited with error code {}.".format(ret))
+        print(f"[FAIL] GLVis exited with error code {ret}.")
         return False
     if not os.path.exists(t_group):
         os.mkdir(t_group)
-    output_name = "{0}/{1}.png".format(t_group, t_name)
+    output_name = f"{t_group0}/{t_name1}.png"
 
-    ret = os.system("mv {0} {1}".format(screenshot_file, output_name))
+    ret = os.system(f"mv {screenshot_file0} {output_name}")
     if ret != 0:
-        print("[FAIL] Could not move output image: exit code {}.".format(ret))
+        print(f"[FAIL] Could not move output image: exit code {ret}.")
         return False
 
     if baseline:
-        baseline_name = "{0}/test.{1}.png".format(baseline, test_name)
+        baseline_name = f"{baseline0}/test.{test_name1}.png"
         return compare_images(baseline_name, output_name)
     else:
         print("[IGNORE] No baseline exists to compare against.")
@@ -106,35 +106,35 @@ def test_stream(exec_path, exec_args, save_file, baseline):
     if exec_args is None:
         exec_args = ""
     test_name = os.path.basename(save_file)
-    print("Testing {}...".format(save_file))
+    print(f"Testing {save_file}...")
 
     # Create new stream file with command to screenshot and close
     stream_data = None
     with open(save_file) as in_f:
         stream_data = in_f.read()
 
-    output_name = "test.{}.png".format(test_name)
-    output_name_fail = "test.fail.{}.png".format(test_name)
+    output_name = f"test.{test_name}.png"
+    output_name_fail = f"test.fail.{test_name}.png"
     tmp_file = "test.saved"
     with open(tmp_file, 'w') as out_f:
         out_f.write(stream_data)
         out_f.write("\nwindow_size 800 600")
-        out_f.write("\nscreenshot {}".format(output_name))
+        out_f.write(f"\nscreenshot {output_name}")
         # Zooming in should create some difference in the images
         out_f.write("\nkeys *")
-        out_f.write("\nscreenshot {}".format(output_name_fail))
+        out_f.write(f"\nscreenshot {output_name_fail}")
         out_f.write("\nkeys q")
 
     # Run GLVis with modified stream file
-    cmd = "{0} {1} -saved {2}".format(exec_path, exec_args, tmp_file)
-    print("Exec: {}".format(cmd))
+    cmd = f"{exec_path} {exec_args} -saved {tmp_file2}"
+    print(f"Exec: {cmd}")
     ret = os.system(cmd)
     if ret != 0:
-        print("[FAIL] GLVis exited with error code {}.".format(ret))
+        print(f"[FAIL] GLVis exited with error code {ret}.")
         return False
 
     if baseline:
-        baseline_name = "{0}/test.{1}.png".format(baseline, test_name)
+        baseline_name = f"{baseline0}/test.{test_name1}.png")
         test_baseline = compare_images(baseline_name, output_name)
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)

From 2bdf1b72fce4b6ccad1c19107262e038d38348f2 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 16:12:41 -0700
Subject: [PATCH 02/52] fix some typos

---
 tests/glvis_driver.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index af3fb2d2..05ff1071 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -78,9 +78,9 @@ def compare_images(baseline_file, output_file, expect_fail=False):
 # Function to test a given glvis command with a variety of key-based commands.
 # Not currently in use.
 def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
-    print(f"Testing {t_group0}:{t_name1}..."))
+    print(f"Testing {t_group}:{t_name}..."))
     full_screenshot_cmd = cmd + screenshot_keys
-    cmd = f"{exec_path0} {exec_args} -k \"{full_screenshot_cmd2}\""
+    cmd = f"{exec_path} {exec_args} -k \"{full_screenshot_cmd}\""
     print(f"Exec: {cmd}")
     ret = os.system(cmd + " > /dev/null 2>&1")
     if ret != 0:
@@ -88,15 +88,15 @@ def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
         return False
     if not os.path.exists(t_group):
         os.mkdir(t_group)
-    output_name = f"{t_group0}/{t_name1}.png"
+    output_name = f"{t_group}/{t_name}.png"
 
-    ret = os.system(f"mv {screenshot_file0} {output_name}")
+    ret = os.system(f"mv {screenshot_file} {output_name}")
     if ret != 0:
         print(f"[FAIL] Could not move output image: exit code {ret}.")
         return False
 
     if baseline:
-        baseline_name = f"{baseline0}/test.{test_name1}.png"
+        baseline_name = f"{baseline}/test.{test_name}.png"
         return compare_images(baseline_name, output_name)
     else:
         print("[IGNORE] No baseline exists to compare against.")
@@ -126,7 +126,7 @@ def test_stream(exec_path, exec_args, save_file, baseline):
         out_f.write("\nkeys q")
 
     # Run GLVis with modified stream file
-    cmd = f"{exec_path} {exec_args} -saved {tmp_file2}"
+    cmd = f"{exec_path} {exec_args} -saved {tmp_file}"
     print(f"Exec: {cmd}")
     ret = os.system(cmd)
     if ret != 0:
@@ -134,7 +134,7 @@ def test_stream(exec_path, exec_args, save_file, baseline):
         return False
 
     if baseline:
-        baseline_name = f"{baseline0}/test.{test_name1}.png")
+        baseline_name = f"{baseline}/test.{test_name}.png")
         test_baseline = compare_images(baseline_name, output_name)
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)

From 6646eaf9c43c9138738b6f1ca7e40cd8bb85b84e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 16:13:06 -0700
Subject: [PATCH 03/52] add requirements.txt

---
 tests/requirements.txt | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 tests/requirements.txt

diff --git a/tests/requirements.txt b/tests/requirements.txt
new file mode 100644
index 00000000..48389795
--- /dev/null
+++ b/tests/requirements.txt
@@ -0,0 +1,3 @@
+scikit-image
+plotly
+numpy >= 1.20.0, < 2.0.0

From f6b6fe455be5f63c52497dc4314553c22a9613e7 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 16:13:55 -0700
Subject: [PATCH 04/52] add fail-fast:false to strategy so tests always
 complete

---
 .github/workflows/builds.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index faa48a8d..2682cf42 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -31,6 +31,7 @@ env:
 jobs:
   builds-and-tests:
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         target: [dbg, opt]

From 968204950ebf4f687989eec471e9f75fc651fc05 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 16:14:19 -0700
Subject: [PATCH 05/52] update workflow to use requirements.txt

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 2682cf42..433022ee 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -231,7 +231,7 @@ jobs:
       if: matrix.build-system == 'cmake'
       run: |
         python -m pip install --upgrade pip
-        pip install scikit-image
+        python -m pip install -r glvis/tests/requirements.txt
 
     - name: setup Linux testing dependencies
       if: matrix.build-system == 'cmake' && matrix.os == 'ubuntu-latest'

From 3e4892094e7f057e8d039ad465c8bf2ad4169127 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 16:22:53 -0700
Subject: [PATCH 06/52] fix a couple more typos

---
 tests/glvis_driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 05ff1071..6772b84e 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -78,7 +78,7 @@ def compare_images(baseline_file, output_file, expect_fail=False):
 # Function to test a given glvis command with a variety of key-based commands.
 # Not currently in use.
 def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
-    print(f"Testing {t_group}:{t_name}..."))
+    print(f"Testing {t_group}:{t_name}...")
     full_screenshot_cmd = cmd + screenshot_keys
     cmd = f"{exec_path} {exec_args} -k \"{full_screenshot_cmd}\""
     print(f"Exec: {cmd}")
@@ -134,7 +134,7 @@ def test_stream(exec_path, exec_args, save_file, baseline):
         return False
 
     if baseline:
-        baseline_name = f"{baseline}/test.{test_name}.png")
+        baseline_name = f"{baseline}/test.{test_name}.png"
         test_baseline = compare_images(baseline_name, output_name)
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)

From 112ec9fa19549eee1e69d52271250801f093f459 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 17:15:03 -0700
Subject: [PATCH 07/52] try adding macos-13 to see if it is more stable

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 433022ee..c9d5fc28 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, macos-13, windows-latest]
         target: [dbg, opt]
         mpi: [seq]
         build-system: [cmake]

From 677bf579245e634e04e95d74fa8e65e4a91cecaf Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 17:31:12 -0700
Subject: [PATCH 08/52] removed .saved from name and fix a typo in an unused
 function

---
 tests/glvis_driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 6772b84e..181b18c9 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -96,7 +96,7 @@ def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
         return False
 
     if baseline:
-        baseline_name = f"{baseline}/test.{test_name}.png"
+        baseline_name = f"{baseline}/test.{t_name}.png"
         return compare_images(baseline_name, output_name)
     else:
         print("[IGNORE] No baseline exists to compare against.")
@@ -105,7 +105,7 @@ def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
 def test_stream(exec_path, exec_args, save_file, baseline):
     if exec_args is None:
         exec_args = ""
-    test_name = os.path.basename(save_file)
+    test_name = os.path.basename(save_file).replace(".saved", "")
     print(f"Testing {save_file}...")
 
     # Create new stream file with command to screenshot and close

From 5187edbe398a45613287d5644d5d03c8414a052e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 17:43:49 -0700
Subject: [PATCH 09/52] update name in cmakelists

---
 tests/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index e869c9c2..6fab8ab9 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -60,10 +60,10 @@ foreach(test_name IN LISTS stream_tests)
       COMMAND ${CMAKE_COMMAND} -E make_directory
         ${CMAKE_CURRENT_SOURCE_DIR}/data/baselines/local
       COMMAND ${CMAKE_COMMAND} -E copy
-        ${CMAKE_CURRENT_BINARY_DIR}/test.${test_name}.saved.png
+        ${CMAKE_CURRENT_BINARY_DIR}/test.${test_name}.png
         ${CMAKE_CURRENT_SOURCE_DIR}/data/baselines/local
       DEPENDS
-        ${CMAKE_CURRENT_BINARY_DIR}/test.${test_name}.saved.png
+        ${CMAKE_CURRENT_BINARY_DIR}/test.${test_name}.png
       VERBATIM)
 
   add_dependencies(rebaseline _rebaseline_stream_${test_name})

From 424d285c45ca3191ec3308c221bb1324825edfeb Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 17:50:40 -0700
Subject: [PATCH 10/52] cleanup. move unused code to test_cmd.py

---
 tests/glvis_driver.py | 83 +++---------------------------------------
 tests/test_cmd.py     | 84 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 78 deletions(-)
 create mode 100644 tests/test_cmd.py

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 181b18c9..2ef59ed1 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -15,38 +15,7 @@
 from skimage.io import imread
 from skimage.metrics import structural_similarity
 
-# Below are key commands that are passed to the -keys command-line argument for
-# glvis in order to perform testing on raw mesh/grid function data (i.e. non-
-# streams).
-#
-# Currently not in use.
-test_cases = {
-    "magnify": "*****",
-    "axes1": "a",
-    "axes2": "aa",
-    "mesh1": "m",
-    "mesh2": "mm",
-    "cut_plane": "i",
-    "cut_plane_rotate": "iyyyy",
-    "cut_plane_rotate_back": "iyyyyYYYY",
-    "cut_plane_transl": "izzzz",
-    "cut_plane_transl_back": "izzzzZZZZ",
-    "orient2d_1": "R",
-    "orient2d_2": "RR",
-    "orient2d_3": "RRR",
-    "orient2d_4": "RRRR",
-    "orient2d_5": "RRRRR",
-    "orient2d_6": "RRRRRR",
-    "orient3d": "Rr",
-    "perspective": "j",
-}
-
-screenshot_keys = "Sq"
-screenshot_file = "GLVis_s01.png"
-
-cutoff_ssim = 0.999
-
-def compare_images(baseline_file, output_file, expect_fail=False):
+def compare_images(baseline_file, output_file, expect_fail=False, CUTOFF_SSIM=0.999):
     # Try to open output image
     output_img = imread(output_file)
     if output_img is None:
@@ -62,7 +31,7 @@ def compare_images(baseline_file, output_file, expect_fail=False):
     # Compare images with SSIM metrics. For two exactly-equal images, SSIM=1.0.
     # We set a cutoff of 0.999 to account for possible differences in rendering.
     ssim = structural_similarity(baseline_img, output_img, channel_axis=2)
-    if ssim < cutoff_ssim:
+    if ssim < CUTOFF_SSIM:
         if expect_fail:
             print("[PASS] Differences were detected in the control case.")
         else:
@@ -72,35 +41,8 @@ def compare_images(baseline_file, output_file, expect_fail=False):
             print("[FAIL] Differences were not detected in the control case.")
         else:
             print("[PASS] Images match.")
-    print(f"       actual ssim = {ssim}, cutoff = {cutoff_ssim}")
-    return ssim >= cutoff_ssim if not expect_fail else ssim < cutoff_ssim
-
-# Function to test a given glvis command with a variety of key-based commands.
-# Not currently in use.
-def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
-    print(f"Testing {t_group}:{t_name}...")
-    full_screenshot_cmd = cmd + screenshot_keys
-    cmd = f"{exec_path} {exec_args} -k \"{full_screenshot_cmd}\""
-    print(f"Exec: {cmd}")
-    ret = os.system(cmd + " > /dev/null 2>&1")
-    if ret != 0:
-        print(f"[FAIL] GLVis exited with error code {ret}.")
-        return False
-    if not os.path.exists(t_group):
-        os.mkdir(t_group)
-    output_name = f"{t_group}/{t_name}.png"
-
-    ret = os.system(f"mv {screenshot_file} {output_name}")
-    if ret != 0:
-        print(f"[FAIL] Could not move output image: exit code {ret}.")
-        return False
-
-    if baseline:
-        baseline_name = f"{baseline}/test.{t_name}.png"
-        return compare_images(baseline_name, output_name)
-    else:
-        print("[IGNORE] No baseline exists to compare against.")
-        return True
+    print(f"       actual ssim = {ssim}, cutoff = {CUTOFF_SSIM}")
+    return ssim >= CUTOFF_SSIM if not expect_fail else ssim < CUTOFF_SSIM
 
 def test_stream(exec_path, exec_args, save_file, baseline):
     if exec_args is None:
@@ -143,21 +85,6 @@ def test_stream(exec_path, exec_args, save_file, baseline):
         print("[IGNORE] No baseline exists to compare against.")
         return True
 
-def test_cmd(exec_path, exec_args, tgroup, baseline):
-    try:
-        os.remove(screenshot_file)
-    except OSError:
-        pass
-    all_tests_passed = True
-    for testname, cmds in test_cases.items():
-        result = test_case(exec_path, exec_args, baseline, tgroup, testname, cmds)
-        all_tests_passed = all_tests_passed and result
-
-    if all_tests_passed:
-        print("All tests passed.")
-    else:
-        sys.exit(1)
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("-s", "--save_stream", help="Path to a GLVis saved stream file.")
@@ -171,4 +98,4 @@ def test_cmd(exec_path, exec_args, tgroup, baseline):
         if not result:
             sys.exit(1)
     else:
-        test_cmd(args.exec_cmd, args.exec_args, args.group_name, args.baseline)
+        raise Exception("test_cmd() is unused. Import from `test_cmd.py`")
diff --git a/tests/test_cmd.py b/tests/test_cmd.py
new file mode 100644
index 00000000..69b5b4a2
--- /dev/null
+++ b/tests/test_cmd.py
@@ -0,0 +1,84 @@
+# Copyright (c) 2010-2024, Lawrence Livermore National Security, LLC. Produced
+# at the Lawrence Livermore National Laboratory. All Rights reserved. See files
+# LICENSE and NOTICE for details. LLNL-CODE-443271.
+#
+# This file is part of the GLVis visualization tool and library. For more
+# information and source code availability see https://glvis.org.
+#
+# GLVis is free software; you can redistribute it and/or modify it under the
+# terms of the BSD-3 license. We welcome feedback and contributions, see file
+# CONTRIBUTING.md for details.
+
+"""
+Code snippets to test glvis in the command-line. None of the code
+contained here is currently being used.
+"""
+
+# Globals
+screenshot_keys = "Sq"
+screenshot_file = "GLVis_s01.png"
+
+# Below are key commands that are passed to the -keys command-line argument for
+# glvis in order to perform testing on raw mesh/grid function data (i.e. non-
+# streams).
+test_cases = {
+    "magnify": "*****",
+    "axes1": "a",
+    "axes2": "aa",
+    "mesh1": "m",
+    "mesh2": "mm",
+    "cut_plane": "i",
+    "cut_plane_rotate": "iyyyy",
+    "cut_plane_rotate_back": "iyyyyYYYY",
+    "cut_plane_transl": "izzzz",
+    "cut_plane_transl_back": "izzzzZZZZ",
+    "orient2d_1": "R",
+    "orient2d_2": "RR",
+    "orient2d_3": "RRR",
+    "orient2d_4": "RRRR",
+    "orient2d_5": "RRRRR",
+    "orient2d_6": "RRRRRR",
+    "orient3d": "Rr",
+    "perspective": "j",
+}
+
+# Function to test a given glvis command with a variety of key-based commands.
+def test_case(exec_path, exec_args, baseline, t_group, t_name, cmd):
+    print(f"Testing {t_group}:{t_name}...")
+    full_screenshot_cmd = cmd + screenshot_keys
+    cmd = f"{exec_path} {exec_args} -k \"{full_screenshot_cmd}\""
+    print(f"Exec: {cmd}")
+    ret = os.system(cmd + " > /dev/null 2>&1")
+    if ret != 0:
+        print(f"[FAIL] GLVis exited with error code {ret}.")
+        return False
+    if not os.path.exists(t_group):
+        os.mkdir(t_group)
+    output_name = f"{t_group}/{t_name}.png"
+
+    ret = os.system(f"mv {screenshot_file} {output_name}")
+    if ret != 0:
+        print(f"[FAIL] Could not move output image: exit code {ret}.")
+        return False
+
+    if baseline:
+        baseline_name = f"{baseline}/test.{t_name}.png"
+        return compare_images(baseline_name, output_name)
+    else:
+        print("[IGNORE] No baseline exists to compare against.")
+        return True
+
+def test_cmd(exec_path, exec_args, tgroup, baseline):
+    try:
+        os.remove(screenshot_file)
+    except OSError:
+        pass
+    all_tests_passed = True
+    for testname, cmds in test_cases.items():
+        result = test_case(exec_path, exec_args, baseline, tgroup, testname, cmds)
+        all_tests_passed = all_tests_passed and result
+
+    if all_tests_passed:
+        print("All tests passed.")
+    else:
+        sys.exit(1)
\ No newline at end of file

From 7673158af3086e323432d482e599f0593461d3b4 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 18:07:07 -0700
Subject: [PATCH 11/52] update baseline_name

---
 tests/glvis_driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 2ef59ed1..d0e4abf2 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -76,7 +76,7 @@ def test_stream(exec_path, exec_args, save_file, baseline):
         return False
 
     if baseline:
-        baseline_name = f"{baseline}/test.{test_name}.png"
+        baseline_name = f"{baseline}/test.{test_name}.saved.png"
         test_baseline = compare_images(baseline_name, output_name)
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)

From 03a67930fdb7f71cbcfec6574b1fec4575e3dba6 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 18:08:06 -0700
Subject: [PATCH 12/52] revert adding macos-13

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index c9d5fc28..433022ee 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, macos-13, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         target: [dbg, opt]
         mpi: [seq]
         build-system: [cmake]

From 3fbf597f8da1ea5ffd02f4a33e25612dc1b4e1ee Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 18:10:30 -0700
Subject: [PATCH 13/52] try forcing build on macos to see if it helps with
 stability

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 433022ee..e30cf087 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -143,7 +143,7 @@ jobs:
     # Also, if we apply to MFEM build the same target as GLVis. This may be
     # superfluous.
     - name: build mfem
-      if: steps.cache-mfem.outputs.cache-hit != 'true'
+      if: steps.cache-mfem.outputs.cache-hit != 'true' || runner.os == 'macos-latest'
       uses: mfem/github-actions/build-mfem@v2.4
       with:
         os: ${{ matrix.os }}

From 2f05fce8208069d034bd6f49e369b43b5903babf Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 18:49:19 -0700
Subject: [PATCH 14/52] (wip) implementing image diff

---
 tests/glvis_driver.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index d0e4abf2..6547b74b 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -12,8 +12,10 @@
 import argparse
 import sys
 import os
+import numpy as np
 from skimage.io import imread
 from skimage.metrics import structural_similarity
+from skimage.color import rgb2gray
 
 def compare_images(baseline_file, output_file, expect_fail=False, CUTOFF_SSIM=0.999):
     # Try to open output image
@@ -44,6 +46,36 @@ def compare_images(baseline_file, output_file, expect_fail=False, CUTOFF_SSIM=0.
     print(f"       actual ssim = {ssim}, cutoff = {CUTOFF_SSIM}")
     return ssim >= CUTOFF_SSIM if not expect_fail else ssim < CUTOFF_SSIM
 
+def color_distance(I1, I2):
+    """
+    L2-norm in rgb space. There are better ways but this is probably good enough.
+    """
+    NORM_CONSTANT = (3*(255**2))**0.5 # max distance
+    l2norm = lambda x: np.linalg.norm(x, ord=2, axis=2)
+    delta = l2norm(I2.astype(int)-I1.astype(int)) / NORM_CONSTANT # output is NxM [0,1]
+    # now we scale to [0,255] and cast as uint8 so it is a "proper" image
+    Idiff_abs = (delta * 255).astype(np.uint8)
+    # get relative version
+    Idiff_rel = (Idiff_abs / Idiff_abs.max()).astype(np.uint8)
+    return {'abs': Idiff_abs,
+            'rel': Idiff_rel,}
+
+
+def generate_image_diff(
+    image1_filename: str,
+    image2_filename: str,
+    imagediff_filename: str,
+) -> None:
+    # Images are read as NxMx3 [uint8] arrays from [0,255]
+    I1 = imread(image1_filename)
+    I2 = imread(image2_filename)
+    # Take absolute "diff"
+    Idiff = color_distance(I1, I2) # output is NxM [0,1]
+    # Illustrate results
+
+
+
+
 def test_stream(exec_path, exec_args, save_file, baseline):
     if exec_args is None:
         exec_args = ""

From f3a51d262a2b9578a50e421a40b8901a6328767d Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 19:12:00 -0700
Subject: [PATCH 15/52] image diff implemented

---
 tests/glvis_driver.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 6547b74b..ab27479f 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -15,7 +15,9 @@
 import numpy as np
 from skimage.io import imread
 from skimage.metrics import structural_similarity
-from skimage.color import rgb2gray
+from skimage.color import rgb2gray, gray2rgb
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
 
 def compare_images(baseline_file, output_file, expect_fail=False, CUTOFF_SSIM=0.999):
     # Try to open output image
@@ -56,11 +58,10 @@ def color_distance(I1, I2):
     # now we scale to [0,255] and cast as uint8 so it is a "proper" image
     Idiff_abs = (delta * 255).astype(np.uint8)
     # get relative version
-    Idiff_rel = (Idiff_abs / Idiff_abs.max()).astype(np.uint8)
+    Idiff_rel = (Idiff_abs / Idiff_abs.max() * 255).astype(np.uint8)
     return {'abs': Idiff_abs,
             'rel': Idiff_rel,}
 
-
 def generate_image_diff(
     image1_filename: str,
     image2_filename: str,
@@ -72,9 +73,13 @@ def generate_image_diff(
     # Take absolute "diff"
     Idiff = color_distance(I1, I2) # output is NxM [0,1]
     # Illustrate results
-
-
-
+    fig = make_subplots(rows=1, cols=3, subplot_titles=('I1', 'I2', 'ΔI (normalized)'))
+    fig.add_trace(go.Image(z=I1), 1, 1)
+    fig.add_trace(go.Image(z=I2), 1, 2)
+    fig.add_trace(go.Heatmap(z=Idiff['rel'], colorscale='inferno'), 1, 3)
+    fig.update_yaxes(autorange='reversed', scaleanchor='x', constrain='domain', row=1, col=3)
+    fig.update_xaxes(constrain='domain', row=1, col=3)
+    fig.write_html(imagediff_filename)
 
 def test_stream(exec_path, exec_args, save_file, baseline):
     if exec_args is None:

From 8f17c633ed67b937500c3a435df83bfc689cb12e Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 19:16:19 -0700
Subject: [PATCH 16/52] runner.os should be matrix.os

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index e30cf087..c0400815 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -143,7 +143,7 @@ jobs:
     # Also, if we apply to MFEM build the same target as GLVis. This may be
     # superfluous.
     - name: build mfem
-      if: steps.cache-mfem.outputs.cache-hit != 'true' || runner.os == 'macos-latest'
+      if: steps.cache-mfem.outputs.cache-hit != 'true' || matrix.os == 'macos-latest'
       uses: mfem/github-actions/build-mfem@v2.4
       with:
         os: ${{ matrix.os }}

From 741c23a527828f1da008f108b4f4f020d30ac6f7 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 19:20:16 -0700
Subject: [PATCH 17/52] testing... remove macos-latest from cache step

---
 .github/workflows/builds.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index c0400815..aee16b01 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -132,6 +132,7 @@ jobs:
     # Install will only run on cache miss.
     - name: cache mfem
       id: cache-mfem
+      if: matrix.os != 'macos-latest'
       uses: actions/cache@v4
       with:
         path: ${{ env.MFEM_TOP_DIR }}

From 18532266da3b8cde6d6ae5a9acff3309982355a2 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 19:32:29 -0700
Subject: [PATCH 18/52] try adding image diff to ci driver

---
 tests/glvis_driver.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index ab27479f..674dfc1b 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -92,8 +92,8 @@ def test_stream(exec_path, exec_args, save_file, baseline):
     with open(save_file) as in_f:
         stream_data = in_f.read()
 
-    output_name = f"test.{test_name}.png"
-    output_name_fail = f"test.fail.{test_name}.png"
+    output_name = f"test.output.{test_name}.png"
+    output_name_fail = f"test.output(zoomed).{test_name}.png"
     tmp_file = "test.saved"
     with open(tmp_file, 'w') as out_f:
         out_f.write(stream_data)
@@ -115,6 +115,7 @@ def test_stream(exec_path, exec_args, save_file, baseline):
     if baseline:
         baseline_name = f"{baseline}/test.{test_name}.saved.png"
         test_baseline = compare_images(baseline_name, output_name)
+        generate_image_diff(baseline_name, output_name, f"test.diff.{test_name}.html")
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)
         return (test_baseline and test_control)

From d12eaa841dde5f3e5368722afb6921198fbb1ca6 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 19:33:09 -0700
Subject: [PATCH 19/52] remove unnecessary (?) extra tar command in test GLVis
 (cmake/linux)

---
 .github/workflows/builds.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index aee16b01..9a0a91cf 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -244,7 +244,6 @@ jobs:
       run: |
         cd glvis && cd build
         xvfb-run -a ctest --verbose
-        tar czvf test_screenshots.tar.gz tests/test.*.png
 
     - name: test GLVis (cmake/mac)
       if: matrix.build-system == 'cmake' && matrix.os == 'macos-latest'

From 6cb85f4e12fe9b33be080e723f36a4905cd6d22c Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 19:35:32 -0700
Subject: [PATCH 20/52] update ci script to upload image diff in tarball

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 9a0a91cf..67572376 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -255,7 +255,7 @@ jobs:
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'
       run: |
         cd glvis && cd build
-        tar czvf test_screenshots.tar.gz tests/test.*.png
+        tar czvf test_screenshots.tar.gz tests/test.*.png tests/test.*.html
 
     - name: upload test screenshots
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'

From eaacb1b1ad9e533efe223f192b65067a292734b6 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 20:43:22 -0700
Subject: [PATCH 21/52] update image diff so zoom/pan is syncronized

---
 tests/glvis_driver.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 674dfc1b..1a60849c 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -73,12 +73,16 @@ def generate_image_diff(
     # Take absolute "diff"
     Idiff = color_distance(I1, I2) # output is NxM [0,1]
     # Illustrate results
-    fig = make_subplots(rows=1, cols=3, subplot_titles=('I1', 'I2', 'ΔI (normalized)'))
+    fig = make_subplots(rows=1, cols=3,
+                        shared_xaxes=True,
+                        shared_yaxes=True,
+                        subplot_titles=('I1', 'I2', 'ΔI (normalized)'))
     fig.add_trace(go.Image(z=I1), 1, 1)
     fig.add_trace(go.Image(z=I2), 1, 2)
-    fig.add_trace(go.Heatmap(z=Idiff['rel'], colorscale='inferno'), 1, 3)
-    fig.update_yaxes(autorange='reversed', scaleanchor='x', constrain='domain', row=1, col=3)
-    fig.update_xaxes(constrain='domain', row=1, col=3)
+    fig.add_trace(go.Image(z=gray2rgb(Idiff['rel'])), 1, 3)
+    # fig.add_trace(go.Heatmap(z=Idiff['rel'], colorscale='inferno'), 1, 3)
+    fig.update_xaxes(matches='x', showticklabels=False, showgrid=False, zeroline=False)
+    fig.update_yaxes(matches='y', showticklabels=False, showgrid=False, zeroline=False)
     fig.write_html(imagediff_filename)
 
 def test_stream(exec_path, exec_args, save_file, baseline):

From 7cdd41810a11e46fe1ef53084103ab8683e7c480 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 23 Jul 2024 20:44:34 -0700
Subject: [PATCH 22/52] for now, also generate image diff for 'zoom' tests so
 we know its working

---
 tests/glvis_driver.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 1a60849c..abf0318d 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -122,6 +122,7 @@ def test_stream(exec_path, exec_args, save_file, baseline):
         generate_image_diff(baseline_name, output_name, f"test.diff.{test_name}.html")
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)
+        generate_image_diff(baseline_name, output_name_fail, f"test.diff(zoomed).{test_name}.html")
         return (test_baseline and test_control)
     else:
         print("[IGNORE] No baseline exists to compare against.")

From 00a54130ead8d084e7211bdb020a6bee03534d6d Mon Sep 17 00:00:00 2001
From: Jan Nikl <nikl1@llnl.gov>
Date: Wed, 24 Jul 2024 11:37:16 -0700
Subject: [PATCH 23/52] A try to fix MacOS runners.

---
 lib/sdl_main.cpp | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/lib/sdl_main.cpp b/lib/sdl_main.cpp
index bede89f4..ecaca130 100644
--- a/lib/sdl_main.cpp
+++ b/lib/sdl_main.cpp
@@ -292,23 +292,23 @@ SdlMainThread::Handle SdlMainThread::GetHandle(SdlWindow* wnd,
       // Since SDL calls aren't guaranteed to be thread-safe, we guard
       // the call to SDL_GL_MakeCurrent.
       lock_guard<mutex> ctx_lock{gl_ctx_mtx};
-#ifdef SDL_VIDEO_DRIVER_COCOA
-      // TODO: Temporary workaround - after merge, everyone should update to
-      // latest SDL
-      SdlCocoaPlatform* mac_platform
-         = dynamic_cast<SdlCocoaPlatform*>(platform.get());
-      if (mac_platform && mac_platform->UseThreadWorkaround())
-      {
-         int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
-         mac_platform->SetCurrentContext(wnd_id);
-      }
-      else
-      {
-         SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-      }
-#else
+      /*#ifdef SDL_VIDEO_DRIVER_COCOA
+            // TODO: Temporary workaround - after merge, everyone should update to
+            // latest SDL
+            SdlCocoaPlatform* mac_platform
+               = dynamic_cast<SdlCocoaPlatform*>(platform.get());
+            if (mac_platform && mac_platform->UseThreadWorkaround())
+            {
+               int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
+               mac_platform->SetCurrentContext(wnd_id);
+            }
+            else
+            {
+               SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
+            }
+      #else*/
       SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-#endif
+      //#endif
    }
    return out_hnd;
 }

From 21a446f5df33b6e56e6afb97a92c4cde01f39627 Mon Sep 17 00:00:00 2001
From: Jan Nikl <nikl1@llnl.gov>
Date: Wed, 24 Jul 2024 11:46:21 -0700
Subject: [PATCH 24/52] Another try to fix MacOS runners.

---
 lib/sdl_main.cpp | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/sdl_main.cpp b/lib/sdl_main.cpp
index ecaca130..a4e04c07 100644
--- a/lib/sdl_main.cpp
+++ b/lib/sdl_main.cpp
@@ -292,23 +292,24 @@ SdlMainThread::Handle SdlMainThread::GetHandle(SdlWindow* wnd,
       // Since SDL calls aren't guaranteed to be thread-safe, we guard
       // the call to SDL_GL_MakeCurrent.
       lock_guard<mutex> ctx_lock{gl_ctx_mtx};
-      /*#ifdef SDL_VIDEO_DRIVER_COCOA
-            // TODO: Temporary workaround - after merge, everyone should update to
-            // latest SDL
-            SdlCocoaPlatform* mac_platform
-               = dynamic_cast<SdlCocoaPlatform*>(platform.get());
-            if (mac_platform && mac_platform->UseThreadWorkaround())
-            {
-               int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
-               mac_platform->SetCurrentContext(wnd_id);
-            }
-            else
-            {
-               SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-            }
-      #else*/
+#ifdef SDL_VIDEO_DRIVER_COCOA
+      // TODO: Temporary workaround - after merge, everyone should update to
+      // latest SDL
+      SdlCocoaPlatform* mac_platform
+         = dynamic_cast<SdlCocoaPlatform*>(platform.get());
+      if (mac_platform)// && mac_platform->UseThreadWorkaround())
+      {
+         PRINT_DEBUG("Using temporary COCOA workaround");
+         int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
+         mac_platform->SetCurrentContext(wnd_id);
+      }
+      else
+      {
+         SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
+      }
+#else
       SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-      //#endif
+#endif
    }
    return out_hnd;
 }

From a033fb463d5799e38144a8bacaa2afcfc2b72f16 Mon Sep 17 00:00:00 2001
From: Jan Nikl <nikl1@llnl.gov>
Date: Wed, 24 Jul 2024 11:51:08 -0700
Subject: [PATCH 25/52] A safer try for MacOS.

---
 lib/sdl_mac.mm   | 5 +++--
 lib/sdl_main.cpp | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/sdl_mac.mm b/lib/sdl_mac.mm
index caa5ff47..c48a8613 100644
--- a/lib/sdl_mac.mm
+++ b/lib/sdl_mac.mm
@@ -65,7 +65,7 @@ inline bool AtLeastVersion(SDL_version sdl_ver, int major, int minor, int patch)
 
 bool SdlCocoaPlatform::UseThreadWorkaround() const
 {
-   static bool first_call = true;
+   /*static bool first_call = true;
    static bool value = false;
    if (first_call)
    {
@@ -74,7 +74,8 @@ inline bool AtLeastVersion(SDL_version sdl_ver, int major, int minor, int patch)
       value = !AtLeastVersion(sdl_ver, 2, 0, 14);
       first_call = false;
    }
-   return value;
+   return value;*/
+   return true;
 }
 
 void SdlCocoaPlatform::ContextUpdate()
diff --git a/lib/sdl_main.cpp b/lib/sdl_main.cpp
index a4e04c07..c594ae07 100644
--- a/lib/sdl_main.cpp
+++ b/lib/sdl_main.cpp
@@ -297,7 +297,7 @@ SdlMainThread::Handle SdlMainThread::GetHandle(SdlWindow* wnd,
       // latest SDL
       SdlCocoaPlatform* mac_platform
          = dynamic_cast<SdlCocoaPlatform*>(platform.get());
-      if (mac_platform)// && mac_platform->UseThreadWorkaround())
+      if (mac_platform && mac_platform->UseThreadWorkaround())
       {
          PRINT_DEBUG("Using temporary COCOA workaround");
          int wnd_id = SDL_GetWindowID(out_hnd.hwnd);

From 81df60b333d2f10bfee66b65e6e2f0f4c1bceab9 Mon Sep 17 00:00:00 2001
From: Jan Nikl <nikl1@llnl.gov>
Date: Wed, 24 Jul 2024 12:12:36 -0700
Subject: [PATCH 26/52] Revert "A safer try for MacOS."

This reverts commit a033fb463d5799e38144a8bacaa2afcfc2b72f16.
---
 lib/sdl_mac.mm   | 5 ++---
 lib/sdl_main.cpp | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/lib/sdl_mac.mm b/lib/sdl_mac.mm
index c48a8613..caa5ff47 100644
--- a/lib/sdl_mac.mm
+++ b/lib/sdl_mac.mm
@@ -65,7 +65,7 @@ inline bool AtLeastVersion(SDL_version sdl_ver, int major, int minor, int patch)
 
 bool SdlCocoaPlatform::UseThreadWorkaround() const
 {
-   /*static bool first_call = true;
+   static bool first_call = true;
    static bool value = false;
    if (first_call)
    {
@@ -74,8 +74,7 @@ inline bool AtLeastVersion(SDL_version sdl_ver, int major, int minor, int patch)
       value = !AtLeastVersion(sdl_ver, 2, 0, 14);
       first_call = false;
    }
-   return value;*/
-   return true;
+   return value;
 }
 
 void SdlCocoaPlatform::ContextUpdate()
diff --git a/lib/sdl_main.cpp b/lib/sdl_main.cpp
index c594ae07..a4e04c07 100644
--- a/lib/sdl_main.cpp
+++ b/lib/sdl_main.cpp
@@ -297,7 +297,7 @@ SdlMainThread::Handle SdlMainThread::GetHandle(SdlWindow* wnd,
       // latest SDL
       SdlCocoaPlatform* mac_platform
          = dynamic_cast<SdlCocoaPlatform*>(platform.get());
-      if (mac_platform && mac_platform->UseThreadWorkaround())
+      if (mac_platform)// && mac_platform->UseThreadWorkaround())
       {
          PRINT_DEBUG("Using temporary COCOA workaround");
          int wnd_id = SDL_GetWindowID(out_hnd.hwnd);

From c8fc30d0359675da7c70f3e94865d9339df67307 Mon Sep 17 00:00:00 2001
From: Jan Nikl <nikl1@llnl.gov>
Date: Wed, 24 Jul 2024 12:13:34 -0700
Subject: [PATCH 27/52] Revert "Another try to fix MacOS runners."

This reverts commit 21a446f5df33b6e56e6afb97a92c4cde01f39627.
---
 lib/sdl_main.cpp | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/lib/sdl_main.cpp b/lib/sdl_main.cpp
index a4e04c07..ecaca130 100644
--- a/lib/sdl_main.cpp
+++ b/lib/sdl_main.cpp
@@ -292,24 +292,23 @@ SdlMainThread::Handle SdlMainThread::GetHandle(SdlWindow* wnd,
       // Since SDL calls aren't guaranteed to be thread-safe, we guard
       // the call to SDL_GL_MakeCurrent.
       lock_guard<mutex> ctx_lock{gl_ctx_mtx};
-#ifdef SDL_VIDEO_DRIVER_COCOA
-      // TODO: Temporary workaround - after merge, everyone should update to
-      // latest SDL
-      SdlCocoaPlatform* mac_platform
-         = dynamic_cast<SdlCocoaPlatform*>(platform.get());
-      if (mac_platform)// && mac_platform->UseThreadWorkaround())
-      {
-         PRINT_DEBUG("Using temporary COCOA workaround");
-         int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
-         mac_platform->SetCurrentContext(wnd_id);
-      }
-      else
-      {
-         SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-      }
-#else
+      /*#ifdef SDL_VIDEO_DRIVER_COCOA
+            // TODO: Temporary workaround - after merge, everyone should update to
+            // latest SDL
+            SdlCocoaPlatform* mac_platform
+               = dynamic_cast<SdlCocoaPlatform*>(platform.get());
+            if (mac_platform && mac_platform->UseThreadWorkaround())
+            {
+               int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
+               mac_platform->SetCurrentContext(wnd_id);
+            }
+            else
+            {
+               SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
+            }
+      #else*/
       SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-#endif
+      //#endif
    }
    return out_hnd;
 }

From 6433a9d43ba951350fdc538e5eb21c84b667a002 Mon Sep 17 00:00:00 2001
From: Jan Nikl <nikl1@llnl.gov>
Date: Wed, 24 Jul 2024 12:13:48 -0700
Subject: [PATCH 28/52] Revert "A try to fix MacOS runners."

This reverts commit 00a54130ead8d084e7211bdb020a6bee03534d6d.
---
 lib/sdl_main.cpp | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/lib/sdl_main.cpp b/lib/sdl_main.cpp
index ecaca130..bede89f4 100644
--- a/lib/sdl_main.cpp
+++ b/lib/sdl_main.cpp
@@ -292,23 +292,23 @@ SdlMainThread::Handle SdlMainThread::GetHandle(SdlWindow* wnd,
       // Since SDL calls aren't guaranteed to be thread-safe, we guard
       // the call to SDL_GL_MakeCurrent.
       lock_guard<mutex> ctx_lock{gl_ctx_mtx};
-      /*#ifdef SDL_VIDEO_DRIVER_COCOA
-            // TODO: Temporary workaround - after merge, everyone should update to
-            // latest SDL
-            SdlCocoaPlatform* mac_platform
-               = dynamic_cast<SdlCocoaPlatform*>(platform.get());
-            if (mac_platform && mac_platform->UseThreadWorkaround())
-            {
-               int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
-               mac_platform->SetCurrentContext(wnd_id);
-            }
-            else
-            {
-               SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-            }
-      #else*/
+#ifdef SDL_VIDEO_DRIVER_COCOA
+      // TODO: Temporary workaround - after merge, everyone should update to
+      // latest SDL
+      SdlCocoaPlatform* mac_platform
+         = dynamic_cast<SdlCocoaPlatform*>(platform.get());
+      if (mac_platform && mac_platform->UseThreadWorkaround())
+      {
+         int wnd_id = SDL_GetWindowID(out_hnd.hwnd);
+         mac_platform->SetCurrentContext(wnd_id);
+      }
+      else
+      {
+         SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
+      }
+#else
       SDL_GL_MakeCurrent(out_hnd.hwnd, out_hnd.gl_ctx);
-      //#endif
+#endif
    }
    return out_hnd;
 }

From 4497f4d7cb2a0c4fd0f4bcf5aa249143b05a4f54 Mon Sep 17 00:00:00 2001
From: Jan Nikl <nikl1@llnl.gov>
Date: Wed, 24 Jul 2024 12:30:49 -0700
Subject: [PATCH 29/52] Fixed init of the main thread.

---
 glvis.cpp   | 9 +++++++++
 lib/sdl.hpp | 1 +
 2 files changed, 10 insertions(+)

diff --git a/glvis.cpp b/glvis.cpp
index 82700368..adf4dc17 100644
--- a/glvis.cpp
+++ b/glvis.cpp
@@ -997,6 +997,8 @@ void PlayScript(istream &scr)
    script = &scr;
    stream_state.keys.clear();
 
+   GetMainThread();
+
    std::thread worker_thread
    {
       [&](StreamState local_state)
@@ -1468,6 +1470,8 @@ int main (int argc, char *argv[])
    // check for saved stream file
    if (stream_file != string_none)
    {
+      GetMainThread();
+
       Session stream_session(stream_state.fix_elem_orient,
                              stream_state.save_coloring);
 
@@ -1527,6 +1531,8 @@ int main (int argc, char *argv[])
    // server mode, read the mesh and the solution from a socket
    if (input == INPUT_SERVER_MODE)
    {
+      GetMainThread();
+
       // Run server in new thread
       std::thread serverThread{GLVisServer, portnum, save_stream,
                                stream_state.fix_elem_orient,
@@ -1560,6 +1566,9 @@ int main (int argc, char *argv[])
          field_type = (use_soln) ? StreamState::FieldType::SCALAR
                       : StreamState::FieldType::MESH;
       }
+
+      GetMainThread();
+
       Session single_session(field_type, std::move(stream_state));
       single_session.StartSession();
 
diff --git a/lib/sdl.hpp b/lib/sdl.hpp
index 6d157328..aef9848a 100644
--- a/lib/sdl.hpp
+++ b/lib/sdl.hpp
@@ -37,6 +37,7 @@ typedef void (*Delegate)();
 typedef bool (*IdleDelegate)();
 
 class SdlMainThread;
+SdlMainThread& GetMainThread();
 
 class SdlWindow
 {

From 85360c77ecbf16d921bb25e87efae82b00a87b54 Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 17:12:00 -0700
Subject: [PATCH 30/52] add type hinting and only generate artifacts on test
 failure

---
 tests/glvis_driver.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index abf0318d..3760c45a 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -13,13 +13,20 @@
 import sys
 import os
 import numpy as np
+from typing import Dict
 from skimage.io import imread
 from skimage.metrics import structural_similarity
 from skimage.color import rgb2gray, gray2rgb
 from plotly.subplots import make_subplots
 import plotly.graph_objects as go
 
-def compare_images(baseline_file, output_file, expect_fail=False, CUTOFF_SSIM=0.999):
+def compare_images(
+    baseline_file: str,
+    output_file: str,
+    expect_fail: bool = False,
+    CUTOFF_SSIM: float = 0.999
+) -> bool:
+
     # Try to open output image
     output_img = imread(output_file)
     if output_img is None:
@@ -48,7 +55,7 @@ def compare_images(baseline_file, output_file, expect_fail=False, CUTOFF_SSIM=0.
     print(f"       actual ssim = {ssim}, cutoff = {CUTOFF_SSIM}")
     return ssim >= CUTOFF_SSIM if not expect_fail else ssim < CUTOFF_SSIM
 
-def color_distance(I1, I2):
+def color_distance(I1: np.array, I2: np.array) -> Dict[str, np.array]:
     """
     L2-norm in rgb space. There are better ways but this is probably good enough.
     """
@@ -70,7 +77,7 @@ def generate_image_diff(
     # Images are read as NxMx3 [uint8] arrays from [0,255]
     I1 = imread(image1_filename)
     I2 = imread(image2_filename)
-    # Take absolute "diff"
+    # Get image diff
     Idiff = color_distance(I1, I2) # output is NxM [0,1]
     # Illustrate results
     fig = make_subplots(rows=1, cols=3,
@@ -85,7 +92,13 @@ def generate_image_diff(
     fig.update_yaxes(matches='y', showticklabels=False, showgrid=False, zeroline=False)
     fig.write_html(imagediff_filename)
 
-def test_stream(exec_path, exec_args, save_file, baseline):
+def test_stream(
+    exec_path: str,
+    exec_args: str,
+    save_file: str,
+    baseline: str
+) -> bool:
+
     if exec_args is None:
         exec_args = ""
     test_name = os.path.basename(save_file).replace(".saved", "")
@@ -119,10 +132,14 @@ def test_stream(exec_path, exec_args, save_file, baseline):
     if baseline:
         baseline_name = f"{baseline}/test.{test_name}.saved.png"
         test_baseline = compare_images(baseline_name, output_name)
-        generate_image_diff(baseline_name, output_name, f"test.diff.{test_name}.html")
+        if not test_baseline:
+            generate_image_diff(baseline_name, output_name,
+                                f"test.diff.{test_name}.html")
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)
-        generate_image_diff(baseline_name, output_name_fail, f"test.diff(zoomed).{test_name}.html")
+        if test_control:
+            generate_image_diff(baseline_name, output_name_fail,
+                                f"test.diff(zoomed).{test_name}.html")
         return (test_baseline and test_control)
     else:
         print("[IGNORE] No baseline exists to compare against.")

From 8be84a0537157d8ee073f01f06f677b4c1d96d23 Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 17:13:46 -0700
Subject: [PATCH 31/52] revert force build for macos

---
 .github/workflows/builds.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 67572376..c500ecab 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -132,7 +132,6 @@ jobs:
     # Install will only run on cache miss.
     - name: cache mfem
       id: cache-mfem
-      if: matrix.os != 'macos-latest'
       uses: actions/cache@v4
       with:
         path: ${{ env.MFEM_TOP_DIR }}
@@ -144,7 +143,7 @@ jobs:
     # Also, if we apply to MFEM build the same target as GLVis. This may be
     # superfluous.
     - name: build mfem
-      if: steps.cache-mfem.outputs.cache-hit != 'true' || matrix.os == 'macos-latest'
+      if: steps.cache-mfem.outputs.cache-hit != 'true'
       uses: mfem/github-actions/build-mfem@v2.4
       with:
         os: ${{ matrix.os }}

From e1e3bcde8b9e75a2e30891df642b335e2a832bb6 Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 17:16:49 -0700
Subject: [PATCH 32/52] typo

---
 tests/glvis_driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 3760c45a..9985d561 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -137,7 +137,7 @@ def test_stream(
                                 f"test.diff.{test_name}.html")
         test_control = compare_images(baseline_name, output_name_fail,
                                       expect_fail=True)
-        if test_control:
+        if not test_control:
             generate_image_diff(baseline_name, output_name_fail,
                                 f"test.diff(zoomed).{test_name}.html")
         return (test_baseline and test_control)

From 3d6302052836e104ab187f383fd0a680e44989ff Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 18:45:46 -0700
Subject: [PATCH 33/52] cleanup

---
 .github/workflows/builds.yml |  2 +-
 tests/glvis_driver.py        | 18 ++++++++----------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index c500ecab..95fc022a 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -254,7 +254,7 @@ jobs:
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'
       run: |
         cd glvis && cd build
-        tar czvf test_screenshots.tar.gz tests/test.*.png tests/test.*.html
+        tar czvf test_screenshots.tar.gz tests/outputs/*
 
     - name: upload test screenshots
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'
diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 9985d561..e07cd6d4 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -79,7 +79,7 @@ def generate_image_diff(
     I2 = imread(image2_filename)
     # Get image diff
     Idiff = color_distance(I1, I2) # output is NxM [0,1]
-    # Illustrate results
+    # Illustrate results as an interactive plotly figure (html)
     fig = make_subplots(rows=1, cols=3,
                         shared_xaxes=True,
                         shared_yaxes=True,
@@ -109,8 +109,9 @@ def test_stream(
     with open(save_file) as in_f:
         stream_data = in_f.read()
 
-    output_name = f"test.output.{test_name}.png"
-    output_name_fail = f"test.output(zoomed).{test_name}.png"
+    output_name = f"outputs/test.nominal.{test_name}.png"
+    output_name_fail = f"outputs/test.zoom.{test_name}.png"
+    output_name_diff = f"outputs/test-diff.{test_name}.html"
     tmp_file = "test.saved"
     with open(tmp_file, 'w') as out_f:
         out_f.write(stream_data)
@@ -133,13 +134,8 @@ def test_stream(
         baseline_name = f"{baseline}/test.{test_name}.saved.png"
         test_baseline = compare_images(baseline_name, output_name)
         if not test_baseline:
-            generate_image_diff(baseline_name, output_name,
-                                f"test.diff.{test_name}.html")
-        test_control = compare_images(baseline_name, output_name_fail,
-                                      expect_fail=True)
-        if not test_control:
-            generate_image_diff(baseline_name, output_name_fail,
-                                f"test.diff(zoomed).{test_name}.html")
+            generate_image_diff(baseline_name, output_name, output_name_diff)
+        test_control = compare_images(baseline_name, output_name_fail, expect_fail=True)
         return (test_baseline and test_control)
     else:
         print("[IGNORE] No baseline exists to compare against.")
@@ -153,6 +149,8 @@ def test_stream(
     parser.add_argument("-n", "--group_name", help="Name of the test group.")
     parser.add_argument("-b", "--baseline", help="Path to test baseline.")
     args = parser.parse_args()
+
+    os.makedirs('outputs')
     if args.save_stream is not None:
         result = test_stream(args.exec_cmd, args.exec_args, args.save_stream, args.baseline)
         if not result:

From 86293024af05f67fa1f25c2449a6ed71d461c2b7 Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 19:14:59 -0700
Subject: [PATCH 34/52] make sure exist_ok=True

---
 tests/glvis_driver.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index e07cd6d4..0507bc7e 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -150,10 +150,12 @@ def test_stream(
     parser.add_argument("-b", "--baseline", help="Path to test baseline.")
     args = parser.parse_args()
 
-    os.makedirs('outputs')
+    # Make a directory for storing test outputs
+    os.makedirs('outputs', exist_ok=True)
+    # Run tests
     if args.save_stream is not None:
         result = test_stream(args.exec_cmd, args.exec_args, args.save_stream, args.baseline)
         if not result:
             sys.exit(1)
     else:
-        raise Exception("test_cmd() is unused. Import from `test_cmd.py`")
+        raise Exception("--save_stream must be specified. test_cmd() is unused. Import from `test_cmd.py`")

From 9cf8f95d1f67a10422ca96472649730cdadfe2cb Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 19:18:16 -0700
Subject: [PATCH 35/52] change subplot titles

---
 tests/glvis_driver.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 0507bc7e..d5066d49 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -73,6 +73,8 @@ def generate_image_diff(
     image1_filename: str,
     image2_filename: str,
     imagediff_filename: str,
+    image1_name: str = "Baseline",
+    image2_name: str = "Test Output",
 ) -> None:
     # Images are read as NxMx3 [uint8] arrays from [0,255]
     I1 = imread(image1_filename)
@@ -83,7 +85,7 @@ def generate_image_diff(
     fig = make_subplots(rows=1, cols=3,
                         shared_xaxes=True,
                         shared_yaxes=True,
-                        subplot_titles=('I1', 'I2', 'ΔI (normalized)'))
+                        subplot_titles=(image1_name, image2_name, 'ΔI (normalized)'))
     fig.add_trace(go.Image(z=I1), 1, 1)
     fig.add_trace(go.Image(z=I2), 1, 2)
     fig.add_trace(go.Image(z=gray2rgb(Idiff['rel'])), 1, 3)

From 73b28a1d429c9f56d75ef781ded01223fd01168c Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 19:51:32 -0700
Subject: [PATCH 36/52] use source= argument for generating html files

---
 tests/glvis_driver.py | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index d5066d49..7383c0c5 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -14,7 +14,8 @@
 import os
 import numpy as np
 from typing import Dict
-from skimage.io import imread
+from base64 import b64encode
+from skimage.io import imread, imsave
 from skimage.metrics import structural_similarity
 from skimage.color import rgb2gray, gray2rgb
 from plotly.subplots import make_subplots
@@ -69,6 +70,12 @@ def color_distance(I1: np.array, I2: np.array) -> Dict[str, np.array]:
     return {'abs': Idiff_abs,
             'rel': Idiff_rel,}
 
+# For the source= argument in plotly
+def get_image_src(filename):
+    with open(filename, "rb") as f:
+        image_bytes = b64encode(f.read()).decode()
+        return f"data:image/png;base64,{image_bytes}"
+
 def generate_image_diff(
     image1_filename: str,
     image2_filename: str,
@@ -79,20 +86,23 @@ def generate_image_diff(
     # Images are read as NxMx3 [uint8] arrays from [0,255]
     I1 = imread(image1_filename)
     I2 = imread(image2_filename)
+
     # Get image diff
-    Idiff = color_distance(I1, I2) # output is NxM [0,1]
+    Idiffs = color_distance(I1, I2) # output is NxM [0,1]
+    Idiff_rel = gray2rgb(Idiffs['rel']) # convert to 3-channel
+    imsave(f"{imagediff_filename}.png", Idiff_rel) # save png to file
+
     # Illustrate results as an interactive plotly figure (html)
     fig = make_subplots(rows=1, cols=3,
                         shared_xaxes=True,
                         shared_yaxes=True,
                         subplot_titles=(image1_name, image2_name, 'ΔI (normalized)'))
-    fig.add_trace(go.Image(z=I1), 1, 1)
-    fig.add_trace(go.Image(z=I2), 1, 2)
-    fig.add_trace(go.Image(z=gray2rgb(Idiff['rel'])), 1, 3)
-    # fig.add_trace(go.Heatmap(z=Idiff['rel'], colorscale='inferno'), 1, 3)
+    fig.add_trace(go.Image(source=get_image_src(image1_filename)), 1, 1)
+    fig.add_trace(go.Image(source=get_image_src(image2_filename)), 1, 2)
+    fig.add_trace(go.Image(source=get_image_src(f"{imagediff_filename}.png")), 1, 3)
     fig.update_xaxes(matches='x', showticklabels=False, showgrid=False, zeroline=False)
     fig.update_yaxes(matches='y', showticklabels=False, showgrid=False, zeroline=False)
-    fig.write_html(imagediff_filename)
+    fig.write_html(f"{imagediff_filename}.html")
 
 def test_stream(
     exec_path: str,
@@ -103,17 +113,19 @@ def test_stream(
 
     if exec_args is None:
         exec_args = ""
-    test_name = os.path.basename(save_file).replace(".saved", "")
     print(f"Testing {save_file}...")
+    test_name = os.path.basename(save_file).replace(".saved", "") # e.g. "ex3"
+    output_dir = f"outputs/{test_name}"
+    os.makedirs(output_dir, exist_ok=True)
 
     # Create new stream file with command to screenshot and close
     stream_data = None
     with open(save_file) as in_f:
         stream_data = in_f.read()
 
-    output_name = f"outputs/test.nominal.{test_name}.png"
-    output_name_fail = f"outputs/test.zoom.{test_name}.png"
-    output_name_diff = f"outputs/test-diff.{test_name}.html"
+    output_name = f"{output_dir}/test.nominal.{test_name}.png"
+    output_name_fail = f"{output_dir}/test.zoom.{test_name}.png"
+    output_name_diff = f"{output_dir}/test-diff.{test_name}"
     tmp_file = "test.saved"
     with open(tmp_file, 'w') as out_f:
         out_f.write(stream_data)
@@ -153,7 +165,7 @@ def test_stream(
     args = parser.parse_args()
 
     # Make a directory for storing test outputs
-    os.makedirs('outputs', exist_ok=True)
+    os.makedirs("outputs", exist_ok=True)
     # Run tests
     if args.save_stream is not None:
         result = test_stream(args.exec_cmd, args.exec_args, args.save_stream, args.baseline)

From caaf3235537d8348ed742466dbd04c36754f8224 Mon Sep 17 00:00:00 2001
From: justinlaughlin <justinglaughlin@gmail.com>
Date: Wed, 24 Jul 2024 20:56:34 -0700
Subject: [PATCH 37/52] remove dir structure from tarball

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 95fc022a..330c3e8a 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -254,7 +254,7 @@ jobs:
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'
       run: |
         cd glvis && cd build
-        tar czvf test_screenshots.tar.gz tests/outputs/*
+        cd tests/outputs && tar czvf ../../test_screenshots.tar.gz .
 
     - name: upload test screenshots
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'

From 2d5c22451472d84cc94e608577d5012291e54fd0 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 12:05:53 -0700
Subject: [PATCH 38/52] fix tarball

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 37e958d5..337301c5 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -254,7 +254,7 @@ jobs:
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'
       run: |
         cd glvis && cd build
-        cd tests/outputs && tar czvf ../../test_screenshots.tar.gz .
+        cd tests && tar czvf ../../test_screenshots.tar.gz outputs
 
     - name: upload test screenshots
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'

From 00c02c1ccdfa8068b72354678c05e95d4176c205 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:09:11 -0700
Subject: [PATCH 39/52] change behavior to always generate diff pngs. generate
 html on failures

---
 tests/glvis_driver.py | 61 +++++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 25 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 2663e1cf..5141bc2c 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -13,7 +13,6 @@
 import sys
 import os
 import numpy as np
-from typing import Dict
 from base64 import b64encode
 from skimage.io import imread, imsave
 from skimage.metrics import structural_similarity
@@ -56,7 +55,7 @@ def compare_images(
     print(f"       actual ssim = {ssim}, cutoff = {CUTOFF_SSIM}")
     return ssim >= CUTOFF_SSIM if not expect_fail else ssim < CUTOFF_SSIM
 
-def color_distance(I1: np.array, I2: np.array) -> Dict[str, np.array]:
+def color_distance(I1: np.array, I2: np.array) -> dict[str, np.array]:
     """
     L2-norm in rgb space. There are better ways but this is probably good enough.
     """
@@ -70,39 +69,46 @@ def color_distance(I1: np.array, I2: np.array) -> Dict[str, np.array]:
     return {'abs': Idiff_abs,
             'rel': Idiff_rel,}
 
-# For the source= argument in plotly
-def get_image_src(filename):
-    with open(filename, "rb") as f:
-        image_bytes = b64encode(f.read()).decode()
-        return f"data:image/png;base64,{image_bytes}"
-
-def generate_image_diff(
+def generate_image_diffs(
     image1_filename: str,
     image2_filename: str,
-    imagediff_filename: str,
-    image1_name: str = "Baseline",
-    image2_name: str = "Test Output",
+    absdiff_filename: str,
+    reldiff_filename: str,
 ) -> None:
     # Images are read as NxMx3 [uint8] arrays from [0,255]
     I1 = imread(image1_filename)
     I2 = imread(image2_filename)
-
-    # Get image diff
+    # Get the image diffs (abs and rel)
     Idiffs = color_distance(I1, I2) # output is NxM [0,1]
-    Idiff_rel = gray2rgb(Idiffs['rel']) # convert to 3-channel
-    imsave(f"{imagediff_filename}.png", Idiff_rel) # save png to file
+    # Save 3-channel image to file
+    imsave(f"{absdiff_filename}.png", gray2rgb(Idiffs['abs']))
+    imsave(f"{reldiff_filename}.png", gray2rgb(Idiffs['rel']))
 
-    # Illustrate results as an interactive plotly figure (html)
-    fig = make_subplots(rows=1, cols=3,
+# For the source= argument in plotly
+def _get_image_src(filename):
+    with open(filename, "rb") as f:
+        image_bytes = b64encode(f.read()).decode()
+        return f"data:image/png;base64,{image_bytes}"
+
+def image_comparison_plot(
+    image_filenames: list[str],
+    image_names: list[str],  # for subtitles
+    output_filename: str,
+):
+    """
+    Illustrate results as an interactive plotly figure (html)
+    """
+    assert len(image_filenames) == len(image_names)
+    n = len(image_filenames)
+    fig = make_subplots(rows=1, cols=n,
                         shared_xaxes=True,
                         shared_yaxes=True,
-                        subplot_titles=(image1_name, image2_name, 'ΔI (normalized)'))
-    fig.add_trace(go.Image(source=get_image_src(image1_filename)), 1, 1)
-    fig.add_trace(go.Image(source=get_image_src(image2_filename)), 1, 2)
-    fig.add_trace(go.Image(source=get_image_src(f"{imagediff_filename}.png")), 1, 3)
+                        subplot_titles=image_filenames)
+    for idx, filename in enumerate(image_filenames):
+        fig.add_trace(go.Image(source=_get_image_src(filename)), 1, idx)
     fig.update_xaxes(matches='x', showticklabels=False, showgrid=False, zeroline=False)
     fig.update_yaxes(matches='y', showticklabels=False, showgrid=False, zeroline=False)
-    fig.write_html(f"{imagediff_filename}.html")
+    fig.write_html(output_filename)
 
 def test_stream(
     exec_path: str,
@@ -125,7 +131,8 @@ def test_stream(
 
     output_name = f"{output_dir}/test.nominal.{test_name}.png"
     output_name_fail = f"{output_dir}/test.zoom.{test_name}.png"
-    output_name_diff = f"{output_dir}/test-diff.{test_name}"
+    absdiff_name = f"{output_dir}/test.nominal.absdiff.{test_name}.png"
+    reldiff_name = f"{output_dir}/test.nominal.reldiff.{test_name}.png"
     tmp_file = "test.saved"
     with open(tmp_file, 'w') as out_f:
         out_f.write(stream_data)
@@ -147,8 +154,12 @@ def test_stream(
     if baseline:
         baseline_name = f"{baseline}/test.{test_name}.saved.png"
         test_baseline = compare_images(baseline_name, output_name)
+        generate_image_diffs(baseline_name, output_name, absdiff_name, reldiff_name)
+        # Generate an interactive html plot, only if the test fails
         if not test_baseline:
-            generate_image_diff(baseline_name, output_name, output_name_diff)
+            image_comparison_plot([baseline_name, output_name, reldiff_name],
+                                  ["Baseline", "Test Output", "Normalized Diff"],
+                                  reldiff_name.replace(".png", ".html"))
         test_control = compare_images(baseline_name, output_name_fail, expect_fail=True)
         return (test_baseline and test_control)
     else:

From 582fd25d5750037ea9981f0d189dacedebf77a5a Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:26:58 -0700
Subject: [PATCH 40/52] fix path for tarball

---
 .github/workflows/builds.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
index 337301c5..4b583aff 100644
--- a/.github/workflows/builds.yml
+++ b/.github/workflows/builds.yml
@@ -254,7 +254,7 @@ jobs:
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'
       run: |
         cd glvis && cd build
-        cd tests && tar czvf ../../test_screenshots.tar.gz outputs
+        cd tests && tar czvf ../test_screenshots.tar.gz outputs
 
     - name: upload test screenshots
       if: always() && matrix.build-system == 'cmake' && matrix.os != 'windows-latest'

From c3dfcdbb1886a2c3fff859d0a728a8b6bf63cebc Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:27:50 -0700
Subject: [PATCH 41/52] fix erroneous extra extension

---
 tests/glvis_driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 5141bc2c..283102e1 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -81,8 +81,8 @@ def generate_image_diffs(
     # Get the image diffs (abs and rel)
     Idiffs = color_distance(I1, I2) # output is NxM [0,1]
     # Save 3-channel image to file
-    imsave(f"{absdiff_filename}.png", gray2rgb(Idiffs['abs']))
-    imsave(f"{reldiff_filename}.png", gray2rgb(Idiffs['rel']))
+    imsave(absdiff_filename", gray2rgb(Idiffs['abs']))
+    imsave(reldiff_filename", gray2rgb(Idiffs['rel']))
 
 # For the source= argument in plotly
 def _get_image_src(filename):

From b56afa309297c9a57f207021b98cac78cf7dc562 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:28:15 -0700
Subject: [PATCH 42/52] typo

---
 tests/glvis_driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 283102e1..c405daca 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -81,8 +81,8 @@ def generate_image_diffs(
     # Get the image diffs (abs and rel)
     Idiffs = color_distance(I1, I2) # output is NxM [0,1]
     # Save 3-channel image to file
-    imsave(absdiff_filename", gray2rgb(Idiffs['abs']))
-    imsave(reldiff_filename", gray2rgb(Idiffs['rel']))
+    imsave(absdiff_filename, gray2rgb(Idiffs['abs']))
+    imsave(reldiff_filename, gray2rgb(Idiffs['rel']))
 
 # For the source= argument in plotly
 def _get_image_src(filename):

From 8073132ef6882defd5c806f0c5ba8b5432d7e625 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:40:41 -0700
Subject: [PATCH 43/52] index bug

---
 tests/glvis_driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index c405daca..e50777f7 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -105,7 +105,7 @@ def image_comparison_plot(
                         shared_yaxes=True,
                         subplot_titles=image_filenames)
     for idx, filename in enumerate(image_filenames):
-        fig.add_trace(go.Image(source=_get_image_src(filename)), 1, idx)
+        fig.add_trace(go.Image(source=_get_image_src(filename)), 1, idx+1)
     fig.update_xaxes(matches='x', showticklabels=False, showgrid=False, zeroline=False)
     fig.update_yaxes(matches='y', showticklabels=False, showgrid=False, zeroline=False)
     fig.write_html(output_filename)

From 3408a23dc45a9bfec673b43bbeb510a49e1b7b4b Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:43:29 -0700
Subject: [PATCH 44/52] use old glvis/data hash for illustration

---
 tests/data | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data b/tests/data
index 7457c82e..81de802a 160000
--- a/tests/data
+++ b/tests/data
@@ -1 +1 @@
-Subproject commit 7457c82e272a521c7877ad54b1ac4e0533945377
+Subproject commit 81de802ad63ba41cd1fcd3825fbb3d2471538ace

From 11784aee93b09d2c0660b748937164d38c7304f4 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:52:30 -0700
Subject: [PATCH 45/52] change subtitles to image_names

---
 tests/glvis_driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index e50777f7..664d047a 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -103,7 +103,7 @@ def image_comparison_plot(
     fig = make_subplots(rows=1, cols=n,
                         shared_xaxes=True,
                         shared_yaxes=True,
-                        subplot_titles=image_filenames)
+                        subplot_titles=image_names)
     for idx, filename in enumerate(image_filenames):
         fig.add_trace(go.Image(source=_get_image_src(filename)), 1, idx+1)
     fig.update_xaxes(matches='x', showticklabels=False, showgrid=False, zeroline=False)

From e91ab5654926585079e514e42140bef6a8718f88 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 13:55:17 -0700
Subject: [PATCH 46/52] revert the glvis/data hash even further for testing

---
 tests/data | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data b/tests/data
index 81de802a..ba8f2f73 160000
--- a/tests/data
+++ b/tests/data
@@ -1 +1 @@
-Subproject commit 81de802ad63ba41cd1fcd3825fbb3d2471538ace
+Subproject commit ba8f2f73ec957059b42bccd95137deac1f8049f9

From 6eeda89977c7b5e505e54f43d422b9abb7e0ee0b Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 14:27:30 -0700
Subject: [PATCH 47/52] include_plotlyjs='cdn' makes html much smaller but
 requires internet

---
 tests/glvis_driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 664d047a..338ae3d0 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -108,7 +108,7 @@ def image_comparison_plot(
         fig.add_trace(go.Image(source=_get_image_src(filename)), 1, idx+1)
     fig.update_xaxes(matches='x', showticklabels=False, showgrid=False, zeroline=False)
     fig.update_yaxes(matches='y', showticklabels=False, showgrid=False, zeroline=False)
-    fig.write_html(output_filename)
+    fig.write_html(output_filename, include_plotlyjs='cdn')
 
 def test_stream(
     exec_path: str,

From 1dea37dfcfbfe7b3159f5235055e2744028248f1 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 14:28:14 -0700
Subject: [PATCH 48/52] since file is small now, always generate diff plot

---
 tests/glvis_driver.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/glvis_driver.py b/tests/glvis_driver.py
index 338ae3d0..c773791e 100644
--- a/tests/glvis_driver.py
+++ b/tests/glvis_driver.py
@@ -156,10 +156,10 @@ def test_stream(
         test_baseline = compare_images(baseline_name, output_name)
         generate_image_diffs(baseline_name, output_name, absdiff_name, reldiff_name)
         # Generate an interactive html plot, only if the test fails
-        if not test_baseline:
-            image_comparison_plot([baseline_name, output_name, reldiff_name],
-                                  ["Baseline", "Test Output", "Normalized Diff"],
-                                  reldiff_name.replace(".png", ".html"))
+        # if not test_baseline:
+        image_comparison_plot([baseline_name, output_name, reldiff_name],
+                                ["Baseline", "Test Output", "Normalized Diff"],
+                                reldiff_name.replace(".png", ".html"))
         test_control = compare_images(baseline_name, output_name_fail, expect_fail=True)
         return (test_baseline and test_control)
     else:

From f2b07888fa3e8e0570e72eee07332870c5f042c2 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Thu, 25 Jul 2024 14:29:36 -0700
Subject: [PATCH 49/52] change glvis/data back to latest version

---
 tests/data | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data b/tests/data
index ba8f2f73..7457c82e 160000
--- a/tests/data
+++ b/tests/data
@@ -1 +1 @@
-Subproject commit ba8f2f73ec957059b42bccd95137deac1f8049f9
+Subproject commit 7457c82e272a521c7877ad54b1ac4e0533945377

From 58cb4fe208826c4933ad60c02b5133c0458ee295 Mon Sep 17 00:00:00 2001
From: Tzanio Kolev <tzanio@llnl.gov>
Date: Tue, 13 Aug 2024 11:32:28 -0700
Subject: [PATCH 50/52] New line

---
 tests/data        | 2 +-
 tests/test_cmd.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/data b/tests/data
index 62d1614c..f15310a9 160000
--- a/tests/data
+++ b/tests/data
@@ -1 +1 @@
-Subproject commit 62d1614cec24f539a0ddd4499250a9343cb20bee
+Subproject commit f15310a941b3dcc61d0d531ed1285898717232a3
diff --git a/tests/test_cmd.py b/tests/test_cmd.py
index 69b5b4a2..b8d2d6e7 100644
--- a/tests/test_cmd.py
+++ b/tests/test_cmd.py
@@ -81,4 +81,4 @@ def test_cmd(exec_path, exec_args, tgroup, baseline):
     if all_tests_passed:
         print("All tests passed.")
     else:
-        sys.exit(1)
\ No newline at end of file
+        sys.exit(1)

From 4b41c584f4765d1cd5f2817b6545af71abbee172 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 13 Aug 2024 12:14:02 -0700
Subject: [PATCH 51/52] update data to most recent

---
 tests/data | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/data b/tests/data
index f15310a9..62d1614c 160000
--- a/tests/data
+++ b/tests/data
@@ -1 +1 @@
-Subproject commit f15310a941b3dcc61d0d531ed1285898717232a3
+Subproject commit 62d1614cec24f539a0ddd4499250a9343cb20bee

From a6ff19456393eb8246d74f7c9041bc535e542c40 Mon Sep 17 00:00:00 2001
From: Justin Laughlin <laughlin6@llnl.gov>
Date: Tue, 13 Aug 2024 12:17:42 -0700
Subject: [PATCH 52/52] update changelog

---
 CHANGELOG | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 8c33a2fc..f175af6b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -14,6 +14,10 @@ Version 4.3.1 (development)
 
 - Fix the Mac binary build in GitHub CI.
 
+- Miscellaneous CI improvements including: generating image diffs for tests,
+  set `fail-fast: false` so that tests always run, rename artifacts to help
+  avoid confusion, code-cleanup/light refactoring.
+
 
 Version 4.3 released on Aug 7, 2024
 ===================================