From 05a3761d988ac82fcad66240eec5947e9cf7b983 Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Sun, 23 Mar 2025 18:01:38 +0100
Subject: [PATCH 1/8] Add extra webserver group for openSUSE

It uses wwwrun (or www group), it's added as last so it shouldn't fail our normal installs.
---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index f5eed5c774..92b0e02116 100644
--- a/configure.ac
+++ b/configure.ac
@@ -101,7 +101,7 @@ if test "x$DOMSERVER_BUILD_ENABLED" = xyes; then
 	if test "x$with_webserver_group" = x; then
 		# Try a number of group names and choose first one found
 		found=0
-		for g in www-data apache httpd nginx ; do
+		for g in www-data apache httpd nginx wwwrun www ; do
 			if getent group $g >/dev/null 2>&1 ; then
 			   found=1
 			   break

From dfb26ede617b688591a9cb5513817a2b75f40555 Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Wed, 26 Mar 2025 21:25:11 +0100
Subject: [PATCH 2/8] Clean all generated docs

sphinx-build adds it's own Makefile which we need to work around or we
can decide to just remove all dirs. This would also remove custom files
someone might have put there.
---
 .github/jobs/data/codespellignorefiles.txt | 1 +
 doc/manual/Makefile                        | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/.github/jobs/data/codespellignorefiles.txt b/.github/jobs/data/codespellignorefiles.txt
index 310e0c3c4d..1b8b222a3d 100644
--- a/.github/jobs/data/codespellignorefiles.txt
+++ b/.github/jobs/data/codespellignorefiles.txt
@@ -27,3 +27,4 @@ composer*
 ./doc/logos
 ./m4
 ./webapp/tests/Unit/Fixtures
+./doc/manual/Makefile
diff --git a/doc/manual/Makefile b/doc/manual/Makefile
index 52b3fd1504..07f6309ed7 100644
--- a/doc/manual/Makefile
+++ b/doc/manual/Makefile
@@ -47,6 +47,10 @@ inplace-uninstall-l:
 clean-l:
 	rm -rf build/doctrees build/team/.doctrees
 	-$(MAKE) -C build/team clean
+	-rm -f build/html/*.html build/html/.buildinfo build/html/objects.inv build/html/searchindex.js
+	-rm -rf build/html/_*
+	-rm -f build/team/*.sty build/team/*.png build/team/*.cls build/team/*.tex build/team/*.xdy
+	-rm -f DOMjudgelogo.pdf latexmkjarc latexmkrc make.bat python.ist
 
 maintainer-clean-l:
 	rm -rf build conf_ref.rst

From f60550224793a38f8b378e58ddc0d9cb3acba5da Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Sat, 15 Mar 2025 15:31:20 +0100
Subject: [PATCH 3/8] Also show the detected C(++) compiler

---
 configure.ac | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/configure.ac b/configure.ac
index 92b0e02116..659183020b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -331,8 +331,10 @@ echo "Summary:"
 echo " * project.............: $PACKAGE_NAME $PACKAGE_VERSION"
 echo " * prefix..............: $prefix"
 if test "x$JUDGEHOST_BUILD_ENABLED" = xyes; then
+echo " * CPP.................: $CPP"
 echo " * CPPFLAGS............: $CPPFLAGS"
 echo " * CFLAGS..............: $CFLAGS"
+echo " * CXX.................: $CXX"
 echo " * CXXFLAGS............: $CXXFLAGS"
 echo " * LDFLAGS.............: $LDFLAGS"
 fi

From d03148dcceb4db9488dbb38fe033a7fa639ba9e0 Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Tue, 18 Mar 2025 19:08:20 +0100
Subject: [PATCH 4/8] Re order to put all variables on top

---
 .github/jobs/configure-checks/all.bats | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/jobs/configure-checks/all.bats b/.github/jobs/configure-checks/all.bats
index 0ac60eae40..6c4ff93a38 100755
--- a/.github/jobs/configure-checks/all.bats
+++ b/.github/jobs/configure-checks/all.bats
@@ -11,6 +11,11 @@ if [ "$distro_id" = "ID=fedora" ]; then
     cmd=dnf
 fi
 
+if [ -z ${test_path+x} ]; then
+    test_path="/domjudge"
+    # Used in the CI
+fi
+
 translate () {
     args="$@"
     if [ "$distro_id" = "ID=fedora" ]; then
@@ -19,11 +24,6 @@ translate () {
     echo "$args"
 }
 
-if [ -z ${test_path+x} ]; then
-    test_path="/domjudge"
-    # Used in the CI
-fi
-
 setup_user() {
     id -u $u || (useradd $u ; groupadd $u || true )>/dev/null
     chown -R $u:$u ./

From 0f2c233b7cd8f823e99e81b15c17857143d67cdd Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Tue, 18 Mar 2025 19:13:00 +0100
Subject: [PATCH 5/8] Fix indentation

---
 .github/jobs/configure-checks/all.bats | 446 ++++++++++++-------------
 1 file changed, 223 insertions(+), 223 deletions(-)

diff --git a/.github/jobs/configure-checks/all.bats b/.github/jobs/configure-checks/all.bats
index 6c4ff93a38..14dc36c71d 100755
--- a/.github/jobs/configure-checks/all.bats
+++ b/.github/jobs/configure-checks/all.bats
@@ -149,12 +149,12 @@ compile_assertions_finished () {
 }
 
 @test "Run as root discouraged" {
-   setup
-   run su root -c "./configure"
-   discourage_root="checking domjudge-user... configure: error: installing/running as root is STRONGLY DISCOURAGED, use --with-domjudge-user=root to override."
-   assert_line "$discourage_root"
-   run su root -c "./configure --with-domjudge-user=root"
-   refute_line "$discourage_root"
+    setup
+    run su root -c "./configure"
+    discourage_root="checking domjudge-user... configure: error: installing/running as root is STRONGLY DISCOURAGED, use --with-domjudge-user=root to override."
+    assert_line "$discourage_root"
+    run su root -c "./configure --with-domjudge-user=root"
+    refute_line "$discourage_root"
 }
 
 @test "Check for missing webserver group" {
@@ -204,257 +204,257 @@ compile_assertions_finished () {
 }
 
 @test "Run as normal user" {
-   setup
-   run ./configure --with-domjudge-user=$u
-   assert_line "checking domjudge-user... $u"
-   run su $u -c "./configure"
-   assert_line "checking domjudge-user... $u (default: current user)"
+    setup
+    run ./configure --with-domjudge-user=$u
+    assert_line "checking domjudge-user... $u"
+    run su $u -c "./configure"
+    assert_line "checking domjudge-user... $u (default: current user)"
 }
 
 @test "cgroup library needed" {
-   cgroup_init_find="checking for cgroup_init in -lcgroup... no"
-   cgroup_init_error="configure: error: Linux cgroup library not found."
-   setup_user
-   repo-install gcc g++
-   repo-remove libcgroup-dev
-   run run_configure
-   assert_line "$cgroup_init_find"
-   assert_line "$cgroup_init_error"
-   repo-install libcgroup-dev
-   run run_configure
-   refute_line "$cgroup_init_find"
-   refute_line "$cgroup_init_error"
+    cgroup_init_find="checking for cgroup_init in -lcgroup... no"
+    cgroup_init_error="configure: error: Linux cgroup library not found."
+    setup_user
+    repo-install gcc g++
+    repo-remove libcgroup-dev
+    run run_configure
+    assert_line "$cgroup_init_find"
+    assert_line "$cgroup_init_error"
+    repo-install libcgroup-dev
+    run run_configure
+    refute_line "$cgroup_init_find"
+    refute_line "$cgroup_init_error"
 }
 
 @test "/opt configured" {
-   setup
-   run run_configure
-   assert_line " * prefix..............: /opt/domjudge"
-   assert_line " * documentation.......: /opt/domjudge/doc"
-   assert_line " * domserver...........: /opt/domjudge/domserver"
-   assert_line "    - bin..............: /opt/domjudge/domserver/bin"
-   assert_line "    - etc..............: /opt/domjudge/domserver/etc"
-   assert_line "    - lib..............: /opt/domjudge/domserver/lib"
-   assert_line "    - log..............: /opt/domjudge/domserver/log"
-   assert_line "    - run..............: /opt/domjudge/domserver/run"
-   assert_line "    - sql..............: /opt/domjudge/domserver/sql"
-   assert_line "    - tmp..............: /opt/domjudge/domserver/tmp"
-   assert_line "    - webapp...........: /opt/domjudge/domserver/webapp"
-   assert_line "    - example_problems.: /opt/domjudge/domserver/example_problems"
-   assert_line " * judgehost...........: /opt/domjudge/judgehost"
-   assert_line "    - bin..............: /opt/domjudge/judgehost/bin"
-   assert_line "    - etc..............: /opt/domjudge/judgehost/etc"
-   assert_line "    - lib..............: /opt/domjudge/judgehost/lib"
-   assert_line "    - libjudge.........: /opt/domjudge/judgehost/lib/judge"
-   assert_line "    - log..............: /opt/domjudge/judgehost/log"
-   assert_line "    - run..............: /opt/domjudge/judgehost/run"
-   assert_line "    - tmp..............: /opt/domjudge/judgehost/tmp"
-   assert_line "    - judge............: /opt/domjudge/judgehost/judgings"
-   assert_line "    - chroot...........: /chroot/domjudge"
+    setup
+    run run_configure
+    assert_line " * prefix..............: /opt/domjudge"
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    assert_line " * domserver...........: /opt/domjudge/domserver"
+    assert_line "    - bin..............: /opt/domjudge/domserver/bin"
+    assert_line "    - etc..............: /opt/domjudge/domserver/etc"
+    assert_line "    - lib..............: /opt/domjudge/domserver/lib"
+    assert_line "    - log..............: /opt/domjudge/domserver/log"
+    assert_line "    - run..............: /opt/domjudge/domserver/run"
+    assert_line "    - sql..............: /opt/domjudge/domserver/sql"
+    assert_line "    - tmp..............: /opt/domjudge/domserver/tmp"
+    assert_line "    - webapp...........: /opt/domjudge/domserver/webapp"
+    assert_line "    - example_problems.: /opt/domjudge/domserver/example_problems"
+    assert_line " * judgehost...........: /opt/domjudge/judgehost"
+    assert_line "    - bin..............: /opt/domjudge/judgehost/bin"
+    assert_line "    - etc..............: /opt/domjudge/judgehost/etc"
+    assert_line "    - lib..............: /opt/domjudge/judgehost/lib"
+    assert_line "    - libjudge.........: /opt/domjudge/judgehost/lib/judge"
+    assert_line "    - log..............: /opt/domjudge/judgehost/log"
+    assert_line "    - run..............: /opt/domjudge/judgehost/run"
+    assert_line "    - tmp..............: /opt/domjudge/judgehost/tmp"
+    assert_line "    - judge............: /opt/domjudge/judgehost/judgings"
+    assert_line "    - chroot...........: /chroot/domjudge"
 }
 
 @test "Prefix configured" {
-   setup
-   run run_configure --prefix=/tmp
-   refute_line " * prefix..............: /opt/domjudge"
-   refute_line " * documentation.......: /opt/domjudge/doc"
-   refute_line " * domserver...........: /opt/domjudge/domserver"
-   refute_line "    - bin..............: /opt/domjudge/domserver/bin"
-   refute_line "    - tmp..............: /opt/domjudge/domserver/tmp"
-   refute_line "    - example_problems.: /opt/domjudge/domserver/example_problems"
-   refute_line " * judgehost...........: /opt/domjudge/judgehost"
-   refute_line "    - libjudge.........: /opt/domjudge/judgehost/lib/judge"
-   refute_line "    - log..............: /opt/domjudge/judgehost/log"
-   refute_line "    - run..............: /opt/domjudge/judgehost/run"
-   refute_line "    - tmp..............: /opt/domjudge/judgehost/tmp"
-   refute_line "    - judge............: /opt/domjudge/judgehost/judgings"
-   assert_line " * prefix..............: /tmp"
-   assert_line " * documentation.......: /tmp/doc"
-   assert_line " * domserver...........: /tmp/domserver"
-   assert_line " * judgehost...........: /tmp/judgehost"
-   assert_line "    - judge............: /tmp/judgehost/judgings"
+    setup
+    run run_configure --prefix=/tmp
+    refute_line " * prefix..............: /opt/domjudge"
+    refute_line " * documentation.......: /opt/domjudge/doc"
+    refute_line " * domserver...........: /opt/domjudge/domserver"
+    refute_line "    - bin..............: /opt/domjudge/domserver/bin"
+    refute_line "    - tmp..............: /opt/domjudge/domserver/tmp"
+    refute_line "    - example_problems.: /opt/domjudge/domserver/example_problems"
+    refute_line " * judgehost...........: /opt/domjudge/judgehost"
+    refute_line "    - libjudge.........: /opt/domjudge/judgehost/lib/judge"
+    refute_line "    - log..............: /opt/domjudge/judgehost/log"
+    refute_line "    - run..............: /opt/domjudge/judgehost/run"
+    refute_line "    - tmp..............: /opt/domjudge/judgehost/tmp"
+    refute_line "    - judge............: /opt/domjudge/judgehost/judgings"
+    assert_line " * prefix..............: /tmp"
+    assert_line " * documentation.......: /tmp/doc"
+    assert_line " * domserver...........: /tmp/domserver"
+    assert_line " * judgehost...........: /tmp/judgehost"
+    assert_line "    - judge............: /tmp/judgehost/judgings"
 }
 
 @test "Check FHS" {
-   setup
-   run run_configure --enable-fhs
-   refute_line " * prefix..............: /opt/domjudge"
-   refute_line " * documentation.......: /opt/domjudge/doc"
-   refute_line " * domserver...........: /opt/domjudge/domserver"
-   refute_line "    - webapp...........: /opt/domjudge/domserver/webapp"
-   refute_line "    - example_problems.: /opt/domjudge/domserver/example_problems"
-   refute_line " * judgehost...........: /opt/domjudge/judgehost"
-   refute_line "    - lib..............: /opt/domjudge/judgehost/lib"
-
-   assert_line " * prefix..............: /usr/local"
-   assert_line " * documentation.......: /usr/local/share/doc/domjudge"
-   assert_line " * domserver...........: "
-   assert_line "    - bin..............: /usr/local/bin"
-   assert_line "    - etc..............: /usr/local/etc/domjudge"
-   assert_line "    - lib..............: /usr/local/lib/domjudge"
-   assert_line "    - log..............: /usr/local/var/log/domjudge"
-   assert_line "    - run..............: /usr/local/var/run/domjudge"
-   assert_line "    - sql..............: /usr/local/share/domjudge/sql"
-   assert_line "    - tmp..............: /tmp"
-   assert_line "    - webapp...........: /usr/local/share/domjudge/webapp"
-   assert_line "    - example_problems.: /usr/local/share/domjudge/example_problems"
-   assert_line " * judgehost...........: "
-   assert_line "    - bin..............: /usr/local/bin"
-   assert_line "    - etc..............: /usr/local/etc/domjudge"
-   assert_line "    - lib..............: /usr/local/lib/domjudge"
-   assert_line "    - libjudge.........: /usr/local/lib/domjudge/judge"
-   assert_line "    - log..............: /usr/local/var/log/domjudge"
-   assert_line "    - run..............: /usr/local/var/run/domjudge"
-   assert_line "    - tmp..............: /tmp"
-   assert_line "    - judge............: /usr/local/var/lib/domjudge/judgings"
-   assert_line "    - chroot...........: /chroot/domjudge"
+    setup
+    run run_configure --enable-fhs
+    refute_line " * prefix..............: /opt/domjudge"
+    refute_line " * documentation.......: /opt/domjudge/doc"
+    refute_line " * domserver...........: /opt/domjudge/domserver"
+    refute_line "    - webapp...........: /opt/domjudge/domserver/webapp"
+    refute_line "    - example_problems.: /opt/domjudge/domserver/example_problems"
+    refute_line " * judgehost...........: /opt/domjudge/judgehost"
+    refute_line "    - lib..............: /opt/domjudge/judgehost/lib"
+
+    assert_line " * prefix..............: /usr/local"
+    assert_line " * documentation.......: /usr/local/share/doc/domjudge"
+    assert_line " * domserver...........: "
+    assert_line "    - bin..............: /usr/local/bin"
+    assert_line "    - etc..............: /usr/local/etc/domjudge"
+    assert_line "    - lib..............: /usr/local/lib/domjudge"
+    assert_line "    - log..............: /usr/local/var/log/domjudge"
+    assert_line "    - run..............: /usr/local/var/run/domjudge"
+    assert_line "    - sql..............: /usr/local/share/domjudge/sql"
+    assert_line "    - tmp..............: /tmp"
+    assert_line "    - webapp...........: /usr/local/share/domjudge/webapp"
+    assert_line "    - example_problems.: /usr/local/share/domjudge/example_problems"
+    assert_line " * judgehost...........: "
+    assert_line "    - bin..............: /usr/local/bin"
+    assert_line "    - etc..............: /usr/local/etc/domjudge"
+    assert_line "    - lib..............: /usr/local/lib/domjudge"
+    assert_line "    - libjudge.........: /usr/local/lib/domjudge/judge"
+    assert_line "    - log..............: /usr/local/var/log/domjudge"
+    assert_line "    - run..............: /usr/local/var/run/domjudge"
+    assert_line "    - tmp..............: /tmp"
+    assert_line "    - judge............: /usr/local/var/lib/domjudge/judgings"
+    assert_line "    - chroot...........: /chroot/domjudge"
 }
 
 @test "Alternative dirs together with FHS" {
-   setup
-   run run_configure --enable-fhs --with-domserver_webappdir=/run/webapp --with-domserver_tmpdir=/tmp/domserver --with-judgehost_tmpdir=/srv/tmp --with-judgehost_judgedir=/srv/judgings --with-judgehost_chrootdir=/srv/chroot/domjudge
-   assert_line " * prefix..............: /usr/local"
-   assert_line " * documentation.......: /usr/local/share/doc/domjudge"
-   assert_line " * domserver...........: "
-   assert_line "    - bin..............: /usr/local/bin"
-   assert_line "    - etc..............: /usr/local/etc/domjudge"
-   assert_line "    - lib..............: /usr/local/lib/domjudge"
-   assert_line "    - log..............: /usr/local/var/log/domjudge"
-   assert_line "    - run..............: /usr/local/var/run/domjudge"
-   assert_line "    - sql..............: /usr/local/share/domjudge/sql"
-   refute_line "    - tmp..............: /tmp"
-   assert_line "    - tmp..............: /tmp/domserver"
-   refute_line "    - webapp...........: /usr/local/share/domjudge/webapp"
-   assert_line "    - webapp...........: /run/webapp"
-   assert_line "    - example_problems.: /usr/local/share/domjudge/example_problems"
-   assert_line " * judgehost...........: "
-   assert_line "    - bin..............: /usr/local/bin"
-   assert_line "    - etc..............: /usr/local/etc/domjudge"
-   assert_line "    - lib..............: /usr/local/lib/domjudge"
-   assert_line "    - libjudge.........: /usr/local/lib/domjudge/judge"
-   assert_line "    - log..............: /usr/local/var/log/domjudge"
-   assert_line "    - run..............: /usr/local/var/run/domjudge"
-   refute_line "    - tmp..............: /tmp"
-   assert_line "    - tmp..............: /srv/tmp"
-   refute_line "    - judge............: /usr/local/var/lib/domjudge/judgings"
-   assert_line "    - judge............: /srv/judgings"
-   refute_line "    - chroot...........: /chroot/domjudge"
-   assert_line "    - chroot...........: /srv/chroot/domjudge"
+    setup
+    run run_configure --enable-fhs --with-domserver_webappdir=/run/webapp --with-domserver_tmpdir=/tmp/domserver --with-judgehost_tmpdir=/srv/tmp --with-judgehost_judgedir=/srv/judgings --with-judgehost_chrootdir=/srv/chroot/domjudge
+    assert_line " * prefix..............: /usr/local"
+    assert_line " * documentation.......: /usr/local/share/doc/domjudge"
+    assert_line " * domserver...........: "
+    assert_line "    - bin..............: /usr/local/bin"
+    assert_line "    - etc..............: /usr/local/etc/domjudge"
+    assert_line "    - lib..............: /usr/local/lib/domjudge"
+    assert_line "    - log..............: /usr/local/var/log/domjudge"
+    assert_line "    - run..............: /usr/local/var/run/domjudge"
+    assert_line "    - sql..............: /usr/local/share/domjudge/sql"
+    refute_line "    - tmp..............: /tmp"
+    assert_line "    - tmp..............: /tmp/domserver"
+    refute_line "    - webapp...........: /usr/local/share/domjudge/webapp"
+    assert_line "    - webapp...........: /run/webapp"
+    assert_line "    - example_problems.: /usr/local/share/domjudge/example_problems"
+    assert_line " * judgehost...........: "
+    assert_line "    - bin..............: /usr/local/bin"
+    assert_line "    - etc..............: /usr/local/etc/domjudge"
+    assert_line "    - lib..............: /usr/local/lib/domjudge"
+    assert_line "    - libjudge.........: /usr/local/lib/domjudge/judge"
+    assert_line "    - log..............: /usr/local/var/log/domjudge"
+    assert_line "    - run..............: /usr/local/var/run/domjudge"
+    refute_line "    - tmp..............: /tmp"
+    assert_line "    - tmp..............: /srv/tmp"
+    refute_line "    - judge............: /usr/local/var/lib/domjudge/judgings"
+    assert_line "    - judge............: /srv/judgings"
+    refute_line "    - chroot...........: /chroot/domjudge"
+    assert_line "    - chroot...........: /srv/chroot/domjudge"
 }
 
 @test "Alternative dirs together with defaults" {
-   setup
-   run run_configure "--with-judgehost_tmpdir=/srv/tmp --with-judgehost_judgedir=/srv/judgings --with-judgehost_chrootdir=/srv/chroot --with-domserver_logdir=/log"
-   assert_line " * prefix..............: /opt/domjudge"
-   assert_line " * documentation.......: /opt/domjudge/doc"
-   assert_line " * domserver...........: /opt/domjudge/domserver"
-   refute_line "    - log..............: /opt/domjudge/domserver/log"
-   assert_line "    - log..............: /log"
-   assert_line " * judgehost...........: /opt/domjudge/judgehost"
-   refute_line "    - tmp..............: /opt/domjudge/judgehost/tmp"
-   assert_line "    - tmp..............: /srv/tmp"
-   refute_line "    - judge............: /opt/domjudge/judgehost/judgings"
-   assert_line "    - judge............: /srv/judgings"
-   refute_line "    - chroot...........: /chroot/domjudge"
-   assert_line "    - chroot...........: /srv/chroot"
+    setup
+    run run_configure "--with-judgehost_tmpdir=/srv/tmp --with-judgehost_judgedir=/srv/judgings --with-judgehost_chrootdir=/srv/chroot --with-domserver_logdir=/log"
+    assert_line " * prefix..............: /opt/domjudge"
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    assert_line " * domserver...........: /opt/domjudge/domserver"
+    refute_line "    - log..............: /opt/domjudge/domserver/log"
+    assert_line "    - log..............: /log"
+    assert_line " * judgehost...........: /opt/domjudge/judgehost"
+    refute_line "    - tmp..............: /opt/domjudge/judgehost/tmp"
+    assert_line "    - tmp..............: /srv/tmp"
+    refute_line "    - judge............: /opt/domjudge/judgehost/judgings"
+    assert_line "    - judge............: /srv/judgings"
+    refute_line "    - chroot...........: /chroot/domjudge"
+    assert_line "    - chroot...........: /srv/chroot"
 }
 
 @test "Default URL not set, docs mention" {
-  setup
-  run run_configure
-  assert_line "checking baseurl... https://example.com/domjudge/"
-  assert_line "Warning: base URL is unconfigured; generating team documentation will"
-  assert_line "not work out of the box!"
-  assert_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
-  assert_line " * website base URL....: https://example.com/domjudge/"
-  assert_line " * documentation.......: /opt/domjudge/doc"
-  run run_configure "--with-baseurl=https://contest.example.org"
-  assert_line "checking baseurl... https://contest.example.org"
-  refute_line "Warning: base URL is unconfigured; generating team documentation will"
-  refute_line "not work out of the box!"
-  refute_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
-  assert_line " * website base URL....: https://contest.example.org"
+    setup
+    run run_configure
+    assert_line "checking baseurl... https://example.com/domjudge/"
+    assert_line "Warning: base URL is unconfigured; generating team documentation will"
+    assert_line "not work out of the box!"
+    assert_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
+    assert_line " * website base URL....: https://example.com/domjudge/"
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    run run_configure "--with-baseurl=https://contest.example.org"
+    assert_line "checking baseurl... https://contest.example.org"
+    refute_line "Warning: base URL is unconfigured; generating team documentation will"
+    refute_line "not work out of the box!"
+    refute_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
+    assert_line " * website base URL....: https://contest.example.org"
 }
 
 @test "Change users" {
-  setup
-  run run_configure
-  assert_line " * default user........: domjudge-bats-user"
-  assert_line " * runguard user.......: domjudge-run"
-  assert_line " * runguard group......: domjudge-run"
-  assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
-  run run_configure "--with-domjudge-user=other_user --with-webserver-group=other_group --with-runuser=other-domjudge-run --with-rungroup=other-run-group"
-  refute_line " * default user........: domjudge-bats-user"
-  refute_line " * runguard user.......: domjudge-run"
-  refute_line " * runguard group......: domjudge-run"
-  for group in www-data apache nginx; do
-    refute_line " * webserver group.....: $group"
-  done
-  assert_line " * default user........: other_user"
-  assert_line " * runguard user.......: other-domjudge-run"
-  assert_line " * runguard group......: other-run-group"
-  assert_line " * webserver group.....: other_group"
+    setup
+    run run_configure
+    assert_line " * default user........: domjudge-bats-user"
+    assert_line " * runguard user.......: domjudge-run"
+    assert_line " * runguard group......: domjudge-run"
+    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
+    run run_configure "--with-domjudge-user=other_user --with-webserver-group=other_group --with-runuser=other-domjudge-run --with-rungroup=other-run-group"
+    refute_line " * default user........: domjudge-bats-user"
+    refute_line " * runguard user.......: domjudge-run"
+    refute_line " * runguard group......: domjudge-run"
+    for group in www-data apache nginx; do
+        refute_line " * webserver group.....: $group"
+    done
+    assert_line " * default user........: other_user"
+    assert_line " * runguard user.......: other-domjudge-run"
+    assert_line " * runguard group......: other-run-group"
+    assert_line " * webserver group.....: other_group"
 }
 
 @test "No docs" {
-  setup
-  run run_configure
-  assert_line " * documentation.......: /opt/domjudge/doc"
-  run run_configure --enable-doc-build
-  assert_line " * documentation.......: /opt/domjudge/doc"
-  run run_configure --disable-doc-build
-  assert_line " * documentation.......: /opt/domjudge/doc (disabled)"
+    setup
+    run run_configure
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    run run_configure --enable-doc-build
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    run run_configure --disable-doc-build
+    assert_line " * documentation.......: /opt/domjudge/doc (disabled)"
 }
 
 @test "Build default (effective host does both domserver & judgehost)" {
-  if [ "$distro_id" = "ID=fedora" ]; then
-      # Fails as libraries are not found
-      skip
-  fi
-  setup
-  run run_configure
-  assert_line " * domserver...........: /opt/domjudge/domserver"
-  assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
-  assert_line " * judgehost...........: /opt/domjudge/judgehost"
-  assert_line " * runguard group......: domjudge-run"
-  run make domserver
-  assert_success
-  run make judgehost
-  assert_success
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # Fails as libraries are not found
+        skip
+    fi
+    setup
+    run run_configure
+    assert_line " * domserver...........: /opt/domjudge/domserver"
+    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
+    assert_line " * judgehost...........: /opt/domjudge/judgehost"
+    assert_line " * runguard group......: domjudge-run"
+    run make domserver
+    assert_success
+    run make judgehost
+    assert_success
 }
 
 @test "Build domserver disabled" {
-  if [ "$distro_id" = "ID=fedora" ]; then
-      # Fails as libraries are not found
-      skip
-  fi
-  setup
-  run run_configure --disable-domserver-build
-  refute_line " * domserver...........: /opt/domjudge/domserver"
-  for group in www-data apache nginx; do
-    refute_line " * webserver group.....: $group"
-  done
-  assert_line " * judgehost...........: /opt/domjudge/judgehost"
-  assert_line " * runguard group......: domjudge-run"
-  run make domserver
-  assert_failure
-  run make judgehost
-  assert_success
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # Fails as libraries are not found
+        skip
+    fi
+    setup
+    run run_configure --disable-domserver-build
+    refute_line " * domserver...........: /opt/domjudge/domserver"
+    for group in www-data apache nginx; do
+        refute_line " * webserver group.....: $group"
+    done
+    assert_line " * judgehost...........: /opt/domjudge/judgehost"
+    assert_line " * runguard group......: domjudge-run"
+    run make domserver
+    assert_failure
+    run make judgehost
+    assert_success
 }
 
 @test "Build judgehost disabled" {
-  if [ "$distro_id" = "ID=fedora" ]; then
-      # Fails as libraries are not found
-      skip
-  fi
-  setup
-  run run_configure --disable-judgehost-build
-  assert_line " * domserver...........: /opt/domjudge/domserver"
-  assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
-  refute_line " * judgehost...........: /opt/domjudge/judgehost"
-  refute_line " * runguard group......: domjudge-run"
-  run make domserver
-  assert_success
-  run make judgehost
-  assert_failure
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # Fails as libraries are not found
+        skip
+    fi
+    setup
+    run run_configure --disable-judgehost-build
+    assert_line " * domserver...........: /opt/domjudge/domserver"
+    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
+    refute_line " * judgehost...........: /opt/domjudge/judgehost"
+    refute_line " * runguard group......: domjudge-run"
+    run make domserver
+    assert_success
+    run make judgehost
+    assert_failure
 }

From 77c187517c30e050b08c5b090d440576be845273 Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Mon, 17 Mar 2025 08:46:56 +0100
Subject: [PATCH 6/8] `setup` is reserved by bats

It's always run before every testcase, which makes that we would always
install certain tools which breaks the tests where we assume to not have
those tools.
---
 .github/jobs/configure-checks/all.bats | 28 +++++++++++++-------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/jobs/configure-checks/all.bats b/.github/jobs/configure-checks/all.bats
index 14dc36c71d..760a4927b3 100755
--- a/.github/jobs/configure-checks/all.bats
+++ b/.github/jobs/configure-checks/all.bats
@@ -29,7 +29,7 @@ setup_user() {
     chown -R $u:$u ./
 }
 
-setup() {
+setup_helper() {
     setup_user
     for shared_file in config.log confdefs.h conftest.err; do
         chmod a+rw $shared_file || true
@@ -149,7 +149,7 @@ compile_assertions_finished () {
 }
 
 @test "Run as root discouraged" {
-    setup
+    setup_helper
     run su root -c "./configure"
     discourage_root="checking domjudge-user... configure: error: installing/running as root is STRONGLY DISCOURAGED, use --with-domjudge-user=root to override."
     assert_line "$discourage_root"
@@ -204,7 +204,7 @@ compile_assertions_finished () {
 }
 
 @test "Run as normal user" {
-    setup
+    setup_helper
     run ./configure --with-domjudge-user=$u
     assert_line "checking domjudge-user... $u"
     run su $u -c "./configure"
@@ -227,7 +227,7 @@ compile_assertions_finished () {
 }
 
 @test "/opt configured" {
-    setup
+    setup_helper
     run run_configure
     assert_line " * prefix..............: /opt/domjudge"
     assert_line " * documentation.......: /opt/domjudge/doc"
@@ -254,7 +254,7 @@ compile_assertions_finished () {
 }
 
 @test "Prefix configured" {
-    setup
+    setup_helper
     run run_configure --prefix=/tmp
     refute_line " * prefix..............: /opt/domjudge"
     refute_line " * documentation.......: /opt/domjudge/doc"
@@ -276,7 +276,7 @@ compile_assertions_finished () {
 }
 
 @test "Check FHS" {
-    setup
+    setup_helper
     run run_configure --enable-fhs
     refute_line " * prefix..............: /opt/domjudge"
     refute_line " * documentation.......: /opt/domjudge/doc"
@@ -311,7 +311,7 @@ compile_assertions_finished () {
 }
 
 @test "Alternative dirs together with FHS" {
-    setup
+    setup_helper
     run run_configure --enable-fhs --with-domserver_webappdir=/run/webapp --with-domserver_tmpdir=/tmp/domserver --with-judgehost_tmpdir=/srv/tmp --with-judgehost_judgedir=/srv/judgings --with-judgehost_chrootdir=/srv/chroot/domjudge
     assert_line " * prefix..............: /usr/local"
     assert_line " * documentation.......: /usr/local/share/doc/domjudge"
@@ -343,7 +343,7 @@ compile_assertions_finished () {
 }
 
 @test "Alternative dirs together with defaults" {
-    setup
+    setup_helper
     run run_configure "--with-judgehost_tmpdir=/srv/tmp --with-judgehost_judgedir=/srv/judgings --with-judgehost_chrootdir=/srv/chroot --with-domserver_logdir=/log"
     assert_line " * prefix..............: /opt/domjudge"
     assert_line " * documentation.......: /opt/domjudge/doc"
@@ -360,7 +360,7 @@ compile_assertions_finished () {
 }
 
 @test "Default URL not set, docs mention" {
-    setup
+    setup_helper
     run run_configure
     assert_line "checking baseurl... https://example.com/domjudge/"
     assert_line "Warning: base URL is unconfigured; generating team documentation will"
@@ -377,7 +377,7 @@ compile_assertions_finished () {
 }
 
 @test "Change users" {
-    setup
+    setup_helper
     run run_configure
     assert_line " * default user........: domjudge-bats-user"
     assert_line " * runguard user.......: domjudge-run"
@@ -397,7 +397,7 @@ compile_assertions_finished () {
 }
 
 @test "No docs" {
-    setup
+    setup_helper
     run run_configure
     assert_line " * documentation.......: /opt/domjudge/doc"
     run run_configure --enable-doc-build
@@ -411,7 +411,7 @@ compile_assertions_finished () {
         # Fails as libraries are not found
         skip
     fi
-    setup
+    setup_helper
     run run_configure
     assert_line " * domserver...........: /opt/domjudge/domserver"
     assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
@@ -428,7 +428,7 @@ compile_assertions_finished () {
         # Fails as libraries are not found
         skip
     fi
-    setup
+    setup_helper
     run run_configure --disable-domserver-build
     refute_line " * domserver...........: /opt/domjudge/domserver"
     for group in www-data apache nginx; do
@@ -447,7 +447,7 @@ compile_assertions_finished () {
         # Fails as libraries are not found
         skip
     fi
-    setup
+    setup_helper
     run run_configure --disable-judgehost-build
     assert_line " * domserver...........: /opt/domjudge/domserver"
     assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"

From 274987efbb0518b54b65b79c3d5e26e02650543c Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Sun, 9 Mar 2025 16:43:52 +0100
Subject: [PATCH 7/8] Build example test release for configure/make tests

Bats testing now works from another directory
---
 .../configure-checks/setup_configure_image.sh |  9 +--
 .../autoconf-check-different-distro.yml       | 23 --------
 .github/workflows/autoconf-check.yml          | 56 ++++++++++++++++++-
 3 files changed, 56 insertions(+), 32 deletions(-)
 delete mode 100644 .github/workflows/autoconf-check-different-distro.yml

diff --git a/.github/jobs/configure-checks/setup_configure_image.sh b/.github/jobs/configure-checks/setup_configure_image.sh
index b3bdcad686..b71dad822c 100755
--- a/.github/jobs/configure-checks/setup_configure_image.sh
+++ b/.github/jobs/configure-checks/setup_configure_image.sh
@@ -7,17 +7,14 @@ distro_id=$(grep "^ID=" /etc/os-release)
 # Install everything for configure and testing
 case $distro_id in
     "ID=fedora")
-        dnf install pkg-config make bats autoconf automake util-linux -y ;;
+        dnf install make bats util-linux -y ;;
     *)
         apt-get update; apt-get full-upgrade -y
-        apt-get install pkg-config make bats autoconf -y ;;
+        apt-get install make bats -y ;;
 esac
 
-# Build the configure file
-make configure
-
 # Install extra assert statements for bots
 cp submit/assert.bash .github/jobs/configure-checks/
 
 # Run the configure tests for this usecase
-test_path="/__w/domjudge/domjudge" bats .github/jobs/configure-checks/all.bats
+test_path="/__w/domjudge/domjudge/release" bats .github/jobs/configure-checks/all.bats
diff --git a/.github/workflows/autoconf-check-different-distro.yml b/.github/workflows/autoconf-check-different-distro.yml
deleted file mode 100644
index 76597171e4..0000000000
--- a/.github/workflows/autoconf-check-different-distro.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: Check autoconf (Other distros)
-on:
-  merge_group:
-  pull_request:
-    branches:
-      - main
-      - '[0-9]+.[0-9]+'
-
-jobs:
-  redhat-family:
-    strategy:
-      matrix:
-        version: [latest]
-        os: [fedora]
-    runs-on: ubuntu-latest
-    container:
-      image: ${{ matrix.os }}:${{ matrix.version }}
-    steps:
-      - name: Install git so we get the .github directory
-        run: dnf install -y git
-      - uses: actions/checkout@v4
-      - name: Setup image and run bats tests
-        run: .github/jobs/configure-checks/setup_configure_image.sh
diff --git a/.github/workflows/autoconf-check.yml b/.github/workflows/autoconf-check.yml
index dee59a67c6..fa4fdcf707 100644
--- a/.github/workflows/autoconf-check.yml
+++ b/.github/workflows/autoconf-check.yml
@@ -7,7 +7,52 @@ on:
       - '[0-9]+.[0-9]+'
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+    container:
+      image: domjudge/gitlabci:24.04
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Create the distribution tarball
+        run: |
+          make dist
+          make distclean
+          cd ..
+          mv domjudge release
+          mkdir domjudge #Workdir for next steps
+          tar -cf release.tar release
+          gzip -9 release.tar
+          mv release.tar.gz domjudge/
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: test-tarball
+          path: ${{ github.workspace }}/release.tar.gz
+
+  redhat-family:
+    needs: build
+    strategy:
+      matrix:
+        version: [latest]
+        os: [fedora]
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.os }}:${{ matrix.version }}
+    steps:
+      - name: Download Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: test-tarball
+      - name: Unpack the "Release" tarball
+        run: tar xvf release.tar.gz
+      - name: Setup image and run bats tests
+        working-directory: release
+        run: .github/jobs/configure-checks/setup_configure_image.sh
+
   debian-family:
+    needs: build
     strategy:
       matrix:
         version: [jammy, focal, rolling]
@@ -28,8 +73,13 @@ jobs:
     container:
       image: ${{ matrix.os }}:${{ matrix.version }}
     steps:
-      - name: Install git so we get the .github directory
-        run: apt-get update; apt-get install -y git
-      - uses: actions/checkout@v4
+      - name: Download Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: test-tarball
+      - name: Unpack the "Release" tarball
+        run: tar xvf release.tar.gz
       - name: Setup image and run bats tests
+        working-directory: release
         run: .github/jobs/configure-checks/setup_configure_image.sh
+

From 5f873f866dcd3228476db37065ab03633afe5f9d Mon Sep 17 00:00:00 2001
From: Michael Vasseur <14887731+vmcj@users.noreply.github.com>
Date: Sun, 9 Mar 2025 18:51:49 +0100
Subject: [PATCH 8/8] Test with other linux distros with new tests

We'll need to start with multiple clean images in future tests so we now
skip certain tests to keep a "clean" image for certain tests and test in
other matrix jobs.

The intent was to both extend the tests and test with different
distrobutions:
- Alpine fails on running bats,
- Fedora has issues with building `runpipe` so fails on `make judgehost`
- Arch needs another package manager to install certain libraries (so is
  left for another PR).
- OpenSUSE lacks some libraries.

The pre-work is already done to start using those images.

make clean should remove what was created by make all
make distclean should remove what was created by configure (plus what
make clean does)
---
 .github/jobs/configure-checks/all.bats        | 566 ++++++++++++++----
 .github/jobs/configure-checks/distclean.bats  | 303 ++++++++++
 .../configure-checks/setup_configure_image.sh |  24 +-
 .github/workflows/autoconf-check.yml          |  92 ++-
 4 files changed, 832 insertions(+), 153 deletions(-)
 create mode 100755 .github/jobs/configure-checks/distclean.bats

diff --git a/.github/jobs/configure-checks/all.bats b/.github/jobs/configure-checks/all.bats
index 760a4927b3..413691ccc5 100755
--- a/.github/jobs/configure-checks/all.bats
+++ b/.github/jobs/configure-checks/all.bats
@@ -3,12 +3,17 @@
 load 'assert'
 
 u="domjudge-bats-user"
-
 distro_id=$(grep "^ID=" /etc/os-release)
 
 cmd="apt-get"
-if [ "$distro_id" = "ID=fedora" ]; then
+if [ "$distro_id" = "ID=alpine" ]; then
+    cmd=apk
+elif [ "$distro_id" = "ID=arch" ]; then
+    cmd=pacman
+elif [ "$distro_id" = "ID=fedora" ]; then
     cmd=dnf
+elif [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+    cmd=zypper
 fi
 
 if [ -z ${test_path+x} ]; then
@@ -16,64 +21,90 @@ if [ -z ${test_path+x} ]; then
     # Used in the CI
 fi
 
+before_files=$(find . -type f)
+
+# Helper functions
 translate () {
-    args="$@"
+    args="$*"
+    if [ "$distro_id" = "ID=arch" ]; then
+        args=${args/g++/}
+        args=${args/libcgroup-dev/libcgroup}
+    fi
+    if [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+        args=${args/g++/gcc-c++}
+        args=${args/libcgroup-dev/libcgroup-devel}
+        args=${args/python3-sphinx/python3-Sphinx}
+        args=${args/python3-sphinx-rtd-theme/python3-sphinx_rtd_theme}
+        args=${args/python3-yaml/python3-PyYAML}
+        args=${args/texgyre/texlive-tex-gyre}
+        args=${args/texlive-latex-extra/texlive-collection-latexextra}
+        args=${args/texlive-latex-recommended/texlive-collection-latexrecommended}
+    fi
     if [ "$distro_id" = "ID=fedora" ]; then
         args=${args/libcgroup-dev/libcgroup-devel}
+        args=${args/tex-gyre/texlive-tex-gyre}
+        args=${args/python3-sphinx-rtd-theme/python3-sphinx_rtd_theme}
+        # Debian/Ubuntu need 2 packages, fedora some more. As we always install those together we just
+        # add them as replacement for and skip the other.
+        args=${args/texlive-latex-recommended/texlive-latex texlive-cmap texlive-metafont texlive-ec texlive-tex-gyre texlive-fncychap texlive-wrapfig texlive-capt-of texlive-framed texlive-upquote texlive-needspace texlive-tabulary texlive-parskip texlive-oberdiek texlive-makeindex}
+        args=${args/texlive-latex-extra/}
     fi
     echo "$args"
 }
 
-setup_user() {
-    id -u $u || (useradd $u ; groupadd $u || true )>/dev/null
-    chown -R $u:$u ./
-}
-
-setup_helper() {
-    setup_user
-    for shared_file in config.log confdefs.h conftest.err; do
-        chmod a+rw $shared_file || true
-    done
-    if [ "$distro_id" = "ID=fedora" ]; then
-        repo-install httpd
+repo-install () {
+    args=$(translate "$@")
+    if [ "$distro_id" = "ID=alpine" ]; then
+        # shellcheck disable=SC2086
+        ${cmd} add $args &>/dev/null
+    elif [ "$distro_id" = "ID=arch" ]; then
+        # shellcheck disable=SC2086
+        ${cmd} -Sy --noconfirm $args >/dev/null
+    else
+        # shellcheck disable=SC2086
+        ${cmd} install -y $args
+	# >/dev/null
     fi
-    repo-install gcc g++ libcgroup-dev composer
-}
-
-run_configure () {
-    su $u -c "./configure $*"
 }
 
-repo-install () {
-    args=$(translate $@)
-    ${cmd} install $args -y >/dev/null
-}
 repo-remove () {
-    args=$(translate $@)
-    ${cmd} remove $args -y #>/dev/null
-    if [ "$distro_id" != "ID=fedora" ]; then
-        apt-get autoremove -y 2>/dev/null
+    args=$(translate "$@")
+    if [ "$distro_id" = "ID=alpine" ]; then
+        # shellcheck disable=SC2086
+        ${cmd} del $args &>/dev/null
+    elif [ "$distro_id" = "ID=arch" ]; then
+        # shellcheck disable=SC2086
+        for arg in $args; do
+            ${cmd} -Rcns "$arg" &>/dev/null || echo "$arg not found (or not removed)"
+        done
+    elif [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+        # shellcheck disable=SC2086
+        ${cmd} remove -y $args &>/dev/null || ret="$?"
+        if [ "$ret" -ne "104" ]; then
+           return $?
+        fi
+    else
+        # shellcheck disable=SC2086
+        ${cmd} remove -y $args &>/dev/null
+    fi
+    if [ "$distro_id" = "ID=debian" ] || [ "$distro_id" = "ID=ubuntu" ]; then
+        apt-get autoremove -y &>/dev/null
     fi
 }
 
-@test "Default empty configure" {
-    # cleanup from earlier runs
-    repo-remove gcc g++ clang
-    run ./configure
-    assert_failure 1
-    assert_line "checking whether configure should try to set CFLAGS... yes"
-    assert_line "checking whether configure should try to set CXXFLAGS... yes"
-    assert_line "checking whether configure should try to set LDFLAGS... yes"
-    assert_line "checking for gcc... no"
-    assert_line "checking for cc... no"
-    assert_line "checking for cl.exe... no"
-    assert_regex "configure: error: in .${test_path}':"
-    assert_line 'configure: error: no acceptable C compiler found in $PATH'
-    assert_regex "See [\`']config.log' for more details"
+docs_required_install () {
+    repo-install python3-sphinx-rtd-theme python3-sphinx rst2pdf tex-gyre latexmk texlive-latex-recommended texlive-latex-extra
+}
+
+run_configure () {
+    su $u -c "./configure $*"
 }
 
 compiler_assertions () {
-    run run_configure
+    # This test is ran with multiple containers, as we only care for the C/C++ test
+    # a bogus value is provided for the webserver as for example arch doesn't have
+    # a default www-data like group.
+    run run_configure --with-webserver-group=not_all_containers_have_www_group
     # Depending on where we run this we might runas wrong user or lack libraries
     # so we can't expect either success or failure.
     assert_line "checking baseurl... https://example.com/domjudge/"
@@ -110,7 +141,169 @@ compile_assertions_finished () {
     assert_line " * LDFLAGS.............:  -fPIE -pie -Wl,-z,relro -Wl,-z,now"
 }
 
-@test "Install GNU C only" {
+setup_user() {
+    uadd=useradd
+    gadd=groupadd
+    if [ "$distro_id" = "ID=alpine" ]; then
+        uadd=adduser
+        gadd=addgroup
+    fi
+    id -u $u >/dev/null || (${uadd} $u ; ${gadd} $u || true )>/dev/null
+    chown -R $u:$u ./
+}
+
+setup_helper() {
+    setup_user
+    for shared_file in config.log confdefs.h conftest.err; do
+        chmod a+rw $shared_file &>/dev/true || true
+    done
+    if [ "$distro_id" = "ID=fedora" ]; then
+        repo-install httpd
+    fi
+    if [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+        repo-install apache2
+    fi
+    repo-install gcc g++ libcgroup-dev composer acl
+}
+
+multiple_compilers() {
+    # all of the tests ran before will have a subset of thos already
+    repo-install gcc g++ clang libcgroup-dev
+    export CC="$1"
+    export CXX="$2"
+    compiler_assertions "$1" "$2"
+    assert_line "checking for gcc... $1"
+    assert_line "checking how to run the C preprocessor... $1 -E"
+    assert_line "checking how to run the C++ preprocessor... $2 -E"
+    assert_line " * CPP.................: $1 -E"
+    assert_line " * CXX.................: $2"
+    compile_assertions_finished
+}
+
+check_docs () {
+    doc_files="domjudge-team-manual.pdf html/index.html team/domjudge-team-manual.pdf"
+    if [ -z "$1" ] || [ "$1" = "before" ]; then
+        for f in $doc_files; do
+            run ls "doc/manual/build/$f"
+            assert_failure
+        done
+    fi
+    if [ -z "$1" ]; then
+        run make docs
+    fi
+    if [ -z "$1" ] || [ "$1" = "after" ]; then
+        for f in $doc_files; do
+            run ls "doc/manual/build/$f"
+            assert_success
+        done
+    fi
+}
+
+build_default() {
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # In the current fedora container it fails with:
+        #/usr/bin/ld: cannot find -lstdc++: No such file or directory
+        #/usr/bin/ld: cannot find -lm: No such file or directory
+        #/usr/bin/ld: cannot find -lc: No such file or directory
+        # even though those libraries are installed and work on a normal
+        # fedora install.
+        skip
+    fi
+    user_make="$1"
+    user_install="$2"
+    if [ -z "$3" ]; then
+        prefix="/opt/domjudge"
+    else
+        prefix="$3"
+    fi
+    make_target="$4"
+    setup_helper
+    run run_configure --prefix="$prefix"
+    assert_line " * domserver...........: $prefix/domserver"
+    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
+    assert_line " * judgehost...........: $prefix/judgehost"
+    assert_line " * runguard group......: domjudge-run"
+    if [ -z "$make_target" ]; then
+        if [ "$user_make" = "root" ]; then
+            run make domserver
+        else
+            run su "$user_make" -c "make domserver"
+        fi
+        assert_success
+        if [ "$user_make" = "root" ]; then
+            run make judgehost
+        else
+            run su "$user_make" -c "make judgehost"
+        fi
+    else
+        if [ "$make_target" = "all" ]; then
+            check_docs "before"
+            run make "$make_target"
+            check_docs "after"
+        fi
+    fi
+    assert_success
+    if [ "$user_install" = "root" ]; then
+        run make install-domserver
+    else
+        run su "$user_install" -c "make install-domserver"
+    fi
+    assert_success
+    run ls "${prefix}"/domserver/webapp
+    assert_success
+    if [ "$user_install" = "root" ]; then
+        run make install-judgehost
+    else
+        run su "$user_install" -c "make install-judgehost"
+    fi
+    assert_success
+    run ls "${prefix}"/judgehost/judgings
+    assert_success
+}
+
+@test "Default empty configure" {
+    # In the default image none of these tools were installed for:
+    # arch, alpine, opensuse, fedora, debian & ubuntu
+    repo-remove gcc g++ clang
+    run ./configure
+    assert_failure 1
+    assert_line "checking whether configure should try to set CFLAGS... yes"
+    assert_line "checking whether configure should try to set CXXFLAGS... yes"
+    assert_line "checking whether configure should try to set LDFLAGS... yes"
+    assert_line "checking for gcc... no"
+    assert_line "checking for cc... no"
+    assert_line "checking for cl.exe... no"
+    assert_regex "configure: error: in .${test_path}':"
+    # shellcheck disable=SC2016
+    assert_line 'configure: error: no acceptable C compiler found in $PATH'
+    assert_regex "See [\`']config.log' for more details"
+}
+
+# All checks need the user already
+setup_user
+
+@test "cgroup library needed" {
+    if [ "${ghtest:?}" != "libcgroup" ]; then
+        skip
+    fi
+    cgroup_init_find="checking for cgroup_init in -lcgroup... no"
+    cgroup_init_error="configure: error: Linux cgroup library not found."
+    setup_user
+    repo-install gcc g++
+    repo-remove libcgroup-dev
+    run run_configure --with-webserver-group=not_every_container_has_www_group
+    assert_line "$cgroup_init_find"
+    assert_line "$cgroup_init_error"
+    repo-install libcgroup-dev
+    run run_configure
+    refute_line "$cgroup_init_find"
+    refute_line "$cgroup_init_error"
+}
+
+@test "Install GNU C only (and libraries)" {
+    if [ "${ghtest:?}" != "gcc" ]; then
+        skip
+    fi
     if [ "$distro_id" = "ID=fedora" ]; then
         # Fedora ships with a gcc with enough C++ support
         skip
@@ -124,6 +317,9 @@ compile_assertions_finished () {
 }
 
 @test "Install GNU C++ only" {
+    if [ "${ghtest:?}" != "g++" ]; then
+        skip
+    fi
     # This does work due to dependencies
     repo-remove clang gcc
     repo-install g++ libcgroup-dev
@@ -134,6 +330,9 @@ compile_assertions_finished () {
 }
 
 @test "Install C/C++ compilers (Clang as alternative)" {
+    if [ "${ghtest:?}" != "clang" ]; then
+        skip
+    fi
     if [ "$distro_id" = "ID=fedora" ]; then
         # Fedora has gcc as dependency for clang
         skip
@@ -148,36 +347,63 @@ compile_assertions_finished () {
     compile_assertions_finished
 }
 
-@test "Run as root discouraged" {
-    setup_helper
-    run su root -c "./configure"
-    discourage_root="checking domjudge-user... configure: error: installing/running as root is STRONGLY DISCOURAGED, use --with-domjudge-user=root to override."
-    assert_line "$discourage_root"
-    run su root -c "./configure --with-domjudge-user=root"
-    refute_line "$discourage_root"
+@test "Select gcc/g++ (multiple installed)" {
+    # all of the above tests will have a subset of this already
+    repo-install gcc g++ clang libcgroup-dev
+    compiler_assertions gcc g++
+    assert_line "checking for gcc... gcc"
+    assert_line "checking for g++... g++"
+    compile_assertions_finished
+}
+
+@test "Select clang alternative [clang/clang] (multiple installed)" {
+    multiple_compilers clang clang
+}
+
+@test "Select clang alternative [cc/c++] (multiple installed)" {
+    multiple_compilers cc c++
+}
+
+@test "Select clang alternative [gcc/g++] (multiple installed)" {
+    multiple_compilers gcc g++
+}
+
+@test "Select clang alternative [clang/g++] (multiple installed)" {
+    multiple_compilers clang g++
+}
+
+@test "Select clang alternative [gcc/clang] (multiple installed)" {
+    multiple_compilers gcc clang
 }
 
 @test "Check for missing webserver group" {
+    # Make sure to not put this test after a test which runs the setup_helper
     if [ "$distro_id" != "ID=fedora" ]; then
         # Debian/Ubuntu start with a www-data group
         skip
     fi
     repo-remove httpd nginx
-    for www_group in nginx apache; do
+    for www_group in nginx apache www wwwrun; do
         userdel ${www_group} || true
         groupdel ${www_group} || true
     done
     run ./configure --with-domjudge-user=$u
     assert_line "checking webserver-group... configure: error: webserver group could not be detected, use --with-webserver-group=GROUP"
+    run ./configure --with-domjudge-user=$u --with-webserver-group=root
+    assert_failure
 }
 
 @test "Check for newly added webserver group (Apache)" {
+    # Just use the testname of a compiler test, to not waste CI time
+    if [ "${ghtest:?}" != "gcc" ]; then
+        skip
+    fi
     if [ "$distro_id" != "ID=fedora" ]; then
         # Debian/Ubuntu start with a www-data group
         skip
     fi
     repo-remove httpd nginx
-    for www_group in nginx apache; do
+    for www_group in nginx apache www wwwrun; do
         userdel ${www_group} || true
         groupdel ${www_group} || true
     done
@@ -188,6 +414,10 @@ compile_assertions_finished () {
 }
 
 @test "Check for newly added webserver group (Nginx)" {
+    # Just use the testname of a compiler test, to not waste CI time
+    if [ "${ghtest:?}" != "g++" ]; then
+        skip
+    fi
     if [ "$distro_id" != "ID=fedora" ]; then
         # Debian/Ubuntu start with a www-data group
         skip
@@ -203,6 +433,19 @@ compile_assertions_finished () {
     assert_line " * webserver group.....: nginx"
 }
 
+@test "Run as root discouraged" {
+    setup_helper
+    run su root -c "./configure"
+    discourage_root="checking domjudge-user... configure: error: installing/running as root is STRONGLY DISCOURAGED, use --with-domjudge-user=root to override."
+    assert_line "$discourage_root"
+    run su root -c "./configure --with-domjudge-user=root"
+    refute_line "$discourage_root"
+    assert_line "checking domjudge-user... root"
+    run su root -c "./configure --with-domjudge-user=$u"
+    refute_line "$discourage_root"
+    assert_line "checking domjudge-user... $u"
+}
+
 @test "Run as normal user" {
     setup_helper
     run ./configure --with-domjudge-user=$u
@@ -211,19 +454,51 @@ compile_assertions_finished () {
     assert_line "checking domjudge-user... $u (default: current user)"
 }
 
-@test "cgroup library needed" {
-    cgroup_init_find="checking for cgroup_init in -lcgroup... no"
-    cgroup_init_error="configure: error: Linux cgroup library not found."
-    setup_user
-    repo-install gcc g++
-    repo-remove libcgroup-dev
+@test "Default URL not set, docs mention" {
+    setup_helper
     run run_configure
-    assert_line "$cgroup_init_find"
-    assert_line "$cgroup_init_error"
-    repo-install libcgroup-dev
+    assert_line "checking baseurl... https://example.com/domjudge/"
+    assert_line "Warning: base URL is unconfigured; generating team documentation will"
+    assert_line "not work out of the box!"
+    assert_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
+    assert_line " * website base URL....: https://example.com/domjudge/"
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    run run_configure "--with-baseurl=https://contest.example.org"
+    assert_line "checking baseurl... https://contest.example.org"
+    refute_line "Warning: base URL is unconfigured; generating team documentation will"
+    refute_line "not work out of the box!"
+    refute_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
+    assert_line " * website base URL....: https://contest.example.org"
+}
+
+@test "Change users" {
+    setup_helper
     run run_configure
-    refute_line "$cgroup_init_find"
-    refute_line "$cgroup_init_error"
+    assert_line " * default user........: domjudge-bats-user"
+    assert_line " * runguard user.......: domjudge-run"
+    assert_line " * runguard group......: domjudge-run"
+    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx|www|wwwrun)$"
+    run run_configure "--with-domjudge-user=other_user --with-webserver-group=other_group --with-runuser=other-domjudge-run --with-rungroup=other-run-group"
+    refute_line " * default user........: domjudge-bats-user"
+    refute_line " * runguard user.......: domjudge-run"
+    refute_line " * runguard group......: domjudge-run"
+    for group in www-data apache nginx; do
+        refute_line " * webserver group.....: $group"
+    done
+    assert_line " * default user........: other_user"
+    assert_line " * runguard user.......: other-domjudge-run"
+    assert_line " * runguard group......: other-run-group"
+    assert_line " * webserver group.....: other_group"
+}
+
+@test "No docs" {
+    setup_helper
+    run run_configure
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    run run_configure --enable-doc-build
+    assert_line " * documentation.......: /opt/domjudge/doc"
+    run run_configure --disable-doc-build
+    assert_line " * documentation.......: /opt/domjudge/doc (disabled)"
 }
 
 @test "/opt configured" {
@@ -359,70 +634,38 @@ compile_assertions_finished () {
     assert_line "    - chroot...........: /srv/chroot"
 }
 
-@test "Default URL not set, docs mention" {
-    setup_helper
-    run run_configure
-    assert_line "checking baseurl... https://example.com/domjudge/"
-    assert_line "Warning: base URL is unconfigured; generating team documentation will"
-    assert_line "not work out of the box!"
-    assert_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
-    assert_line " * website base URL....: https://example.com/domjudge/"
-    assert_line " * documentation.......: /opt/domjudge/doc"
-    run run_configure "--with-baseurl=https://contest.example.org"
-    assert_line "checking baseurl... https://contest.example.org"
-    refute_line "Warning: base URL is unconfigured; generating team documentation will"
-    refute_line "not work out of the box!"
-    refute_line "Rerun configure with option '--with-baseurl=BASEURL' to correct this."
-    assert_line " * website base URL....: https://contest.example.org"
+@test "Build default [root/root] (effective host does both domserver & judgehost)" {
+    build_default root root
 }
 
-@test "Change users" {
-    setup_helper
-    run run_configure
-    assert_line " * default user........: domjudge-bats-user"
-    assert_line " * runguard user.......: domjudge-run"
-    assert_line " * runguard group......: domjudge-run"
-    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
-    run run_configure "--with-domjudge-user=other_user --with-webserver-group=other_group --with-runuser=other-domjudge-run --with-rungroup=other-run-group"
-    refute_line " * default user........: domjudge-bats-user"
-    refute_line " * runguard user.......: domjudge-run"
-    refute_line " * runguard group......: domjudge-run"
-    for group in www-data apache nginx; do
-        refute_line " * webserver group.....: $group"
-    done
-    assert_line " * default user........: other_user"
-    assert_line " * runguard user.......: other-domjudge-run"
-    assert_line " * runguard group......: other-run-group"
-    assert_line " * webserver group.....: other_group"
+# Cleanup for next runs
+rm -rf /opt/domjudge >/dev/zero
+make clean >/dev/zero
+
+@test "Build default [user/root] (effective host does both domserver & judgehost)" {
+    build_default "$u" root
 }
 
-@test "No docs" {
-    setup_helper
-    run run_configure
-    assert_line " * documentation.......: /opt/domjudge/doc"
-    run run_configure --enable-doc-build
-    assert_line " * documentation.......: /opt/domjudge/doc"
-    run run_configure --disable-doc-build
-    assert_line " * documentation.......: /opt/domjudge/doc (disabled)"
+# Cleanup for next runs
+rm -rf /opt/domjudge &>/dev/zero
+make clean &>/dev/zero
+
+@test "Build default [user/user] (effective host does both domserver & judgehost)" {
+    build_default "$u" "$u" /tmp/domjudge
 }
 
-@test "Build default (effective host does both domserver & judgehost)" {
-    if [ "$distro_id" = "ID=fedora" ]; then
-        # Fails as libraries are not found
-        skip
-    fi
-    setup_helper
-    run run_configure
-    assert_line " * domserver...........: /opt/domjudge/domserver"
-    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
-    assert_line " * judgehost...........: /opt/domjudge/judgehost"
-    assert_line " * runguard group......: domjudge-run"
-    run make domserver
-    assert_success
-    run make judgehost
-    assert_success
+# Cleanup for next runs
+rm -rf /opt/domjudge &>/dev/zero
+make clean &>/dev/zero
+
+@test "Build default (effective host does everything)" {
+    build_default root root "" build
 }
 
+# Cleanup for next runs
+rm -rf /opt/domjudge &>/dev/zero
+make clean &>/dev/zero
+
 @test "Build domserver disabled" {
     if [ "$distro_id" = "ID=fedora" ]; then
         # Fails as libraries are not found
@@ -440,21 +683,84 @@ compile_assertions_finished () {
     assert_failure
     run make judgehost
     assert_success
+    run make install-judgehost
+    assert_success
+    run ls /opt/domjudge/judgehost/judgings
+    assert_success
 }
 
-@test "Build judgehost disabled" {
-    if [ "$distro_id" = "ID=fedora" ]; then
-        # Fails as libraries are not found
-        skip
-    fi
+# Cleanup for next runs
+rm -rf /opt/domjudge &>/dev/zero
+make clean &>/dev/zero
+
+@test "Build judgehost disabled [user/root]" {
     setup_helper
-    run run_configure --disable-judgehost-build
-    assert_line " * domserver...........: /opt/domjudge/domserver"
+    run run_configure --disable-judgehost-build --prefix=/tmp/domjudge
+    assert_line " * domserver...........: /tmp/domjudge/domserver"
     assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
-    refute_line " * judgehost...........: /opt/domjudge/judgehost"
+    refute_line " * judgehost...........: /tmp/domjudge/judgehost"
     refute_line " * runguard group......: domjudge-run"
-    run make domserver
+    run su $u -c "make domserver 2>&1"
     assert_success
-    run make judgehost
+    run make install-domserver
+    assert_success
+    run ls /tmp/domjudge/domserver/webapp
+    assert_success
+    run su $u -c "make judgehost"
+    assert_failure
+}
+
+# Cleanup for next runs
+rm -rf /tmp/domjudge &>/dev/zero
+make clean &>/dev/zero
+
+@test "Documentation generation works" {
+    docs_required_install
+    setup_helper
+    run run_configure
+    check_docs
+    make clean
+    run run_configure --with-baseurl=http://domjudge/alias
+    check_docs
+}
+
+# Cleanup for next runs
+make clean &>/dev/zero
+
+@test "Build all (effective host does everything, also docs)" {
+    build_default root root "" all
+}
+
+# Cleanup for next runs
+rm -rf /opt/domjudge &>/dev/zero
+make clean &>/dev/zero
+
+@test "Running make clean shows no permission errors" {
+    docs_required_install
+    setup_helper
+    run run_configure
+    assert_line " * domserver...........: /opt/domjudge/domserver"
+    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
+    assert_line " * judgehost...........: /opt/domjudge/judgehost"
+    run make all
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # Because compiling fails
+        assert_failure
+    else
+        assert_success
+    fi
+    run make install-domserver
+    assert_success
+    run make install-judgehost
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # Because compiling fails
+        assert_failure
+    else
+        assert_success
+    fi
+    run make install-docs
+    assert_success
+    run make clean
+    # Currently there is still an error
     assert_failure
 }
diff --git a/.github/jobs/configure-checks/distclean.bats b/.github/jobs/configure-checks/distclean.bats
new file mode 100755
index 0000000000..9b474c061b
--- /dev/null
+++ b/.github/jobs/configure-checks/distclean.bats
@@ -0,0 +1,303 @@
+#!/usr/bin/env bats
+
+load 'assert'
+
+u="domjudge-bats-user"
+distro_id=$(grep "^ID=" /etc/os-release)
+
+cmd="apt-get"
+if [ "$distro_id" = "ID=alpine" ]; then
+    cmd=apk
+elif [ "$distro_id" = "ID=arch" ]; then
+    cmd=pacman
+elif [ "$distro_id" = "ID=fedora" ]; then
+    cmd=dnf
+elif [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+    cmd=zypper
+fi
+
+if [ -z ${test_path+x} ]; then
+    test_path="/domjudge"
+    # Used in the CI
+fi
+
+before_files=$(find . -type f)
+
+# Helper functions
+translate () {
+    args="$*"
+    if [ "$distro_id" = "ID=arch" ]; then
+        args=${args/g++/}
+        args=${args/libcgroup-dev/libcgroup}
+    fi
+    if [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+        args=${args/g++/gcc-c++}
+        args=${args/libcgroup-dev/libcgroup-devel}
+        args=${args/python3-sphinx/python3-Sphinx}
+        args=${args/python3-sphinx-rtd-theme/python3-sphinx_rtd_theme}
+        args=${args/python3-yaml/python3-PyYAML}
+        args=${args/texgyre/texlive-tex-gyre}
+        args=${args/texlive-latex-extra/texlive-collection-latexextra}
+        args=${args/texlive-latex-recommended/texlive-collection-latexrecommended}
+    fi
+    if [ "$distro_id" = "ID=fedora" ]; then
+        args=${args/libcgroup-dev/libcgroup-devel}
+        args=${args/tex-gyre/texlive-tex-gyre}
+        args=${args/python3-sphinx-rtd-theme/python3-sphinx_rtd_theme}
+        # Debian/Ubuntu need 2 packages, fedora some more. As we always install those together we just
+        # add them as replacement for and skip the other.
+        args=${args/texlive-latex-recommended/texlive-latex texlive-cmap texlive-metafont texlive-ec texlive-tex-gyre texlive-fncychap texlive-wrapfig texlive-capt-of texlive-framed texlive-upquote texlive-needspace texlive-tabulary texlive-parskip texlive-oberdiek texlive-makeindex}
+        args=${args/texlive-latex-extra/}
+    fi
+    echo "$args"
+}
+
+repo-install () {
+    args=$(translate "$@")
+    if [ "$distro_id" = "ID=alpine" ]; then
+        # shellcheck disable=SC2086
+        ${cmd} add $args &>/dev/null
+    elif [ "$distro_id" = "ID=arch" ]; then
+        # shellcheck disable=SC2086
+        ${cmd} -Sy --noconfirm $args >/dev/null
+    else
+        # shellcheck disable=SC2086
+        ${cmd} install -y $args
+	# >/dev/null
+    fi
+}
+
+repo-remove () {
+    args=$(translate "$@")
+    if [ "$distro_id" = "ID=alpine" ]; then
+        # shellcheck disable=SC2086
+        ${cmd} del $args &>/dev/null
+    elif [ "$distro_id" = "ID=arch" ]; then
+        # shellcheck disable=SC2086
+        for arg in $args; do
+            ${cmd} -Rcns "$arg" &>/dev/null || echo "$arg not found (or not removed)"
+        done
+    elif [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+        # shellcheck disable=SC2086
+        ${cmd} remove -y $args &>/dev/null || ret="$?"
+        if [ "$ret" -ne "104" ]; then
+           return $?
+        fi
+    else
+        # shellcheck disable=SC2086
+        ${cmd} remove -y $args &>/dev/null
+    fi
+    if [ "$distro_id" = "ID=debian" ] || [ "$distro_id" = "ID=ubuntu" ]; then
+        apt-get autoremove -y &>/dev/null
+    fi
+}
+
+docs_required_install () {
+    repo-install python3-sphinx-rtd-theme python3-sphinx rst2pdf tex-gyre latexmk texlive-latex-recommended texlive-latex-extra
+}
+
+run_configure () {
+    su $u -c "./configure $*"
+}
+
+compiler_assertions () {
+    # This test is ran with multiple containers, as we only care for the C/C++ test
+    # a bogus value is provided for the webserver as for example arch doesn't have
+    # a default www-data like group.
+    run run_configure --with-webserver-group=not_all_containers_have_www_group
+    # Depending on where we run this we might runas wrong user or lack libraries
+    # so we can't expect either success or failure.
+    assert_line "checking baseurl... https://example.com/domjudge/"
+    assert_line "checking whether configure should try to set CFLAGS... yes"
+    assert_line "checking whether configure should try to set CXXFLAGS... yes"
+    assert_line "checking whether configure should try to set LDFLAGS... yes"
+    assert_line "checking whether the C compiler works... yes"
+    assert_line "checking for C compiler default output file name... a.out"
+    assert_line "checking for suffix of executables... "
+    assert_line "checking whether we are cross compiling... no"
+    assert_line "checking for suffix of object files... o"
+    assert_regex "checking whether .*GNU C.*\.\.\. yes"
+    assert_line "checking whether C compiler accepts -Wall... yes"
+    assert_line "checking whether C compiler accepts -fstack-protector... yes"
+    assert_line "checking whether C compiler accepts -fPIE... yes"
+    assert_line "checking whether C compiler accepts -D_FORTIFY_SOURCE=2... yes"
+    assert_line "checking whether the linker accepts -fPIE... yes"
+    assert_line "checking whether the linker accepts -pie... yes"
+    assert_line "checking whether the linker accepts -Wl,-z,relro... yes"
+    assert_line "checking whether the linker accepts -Wl,-z,now... yes"
+    assert_line "checking whether $1 accepts -g... yes"
+    assert_regex "^checking for $1 option to (enable C11 features|accept ISO C89)\.\.\. none needed$"
+    assert_line "checking whether $1 accepts -g... (cached) yes"
+    if [ -n "$2" ]; then
+        assert_line "checking whether $2 accepts -g... yes"
+        assert_line "checking how to run the C preprocessor... $1 -E"
+        assert_line "checking how to run the C++ preprocessor... $2 -E"
+    fi
+}
+
+compile_assertions_finished () {
+    assert_line " * CFLAGS..............: -g -O2 -Wall -Wformat -Wformat-security -pedantic -fstack-protector -fPIE -D_FORTIFY_SOURCE=2 -std=c11"
+    assert_line " * CXXFLAGS............: -g -O2 -Wall -Wformat -Wformat-security -pedantic -fstack-protector -fPIE -D_FORTIFY_SOURCE=2 -std=c++11"
+    assert_line " * LDFLAGS.............:  -fPIE -pie -Wl,-z,relro -Wl,-z,now"
+}
+
+setup_user() {
+    uadd=useradd
+    gadd=groupadd
+    if [ "$distro_id" = "ID=alpine" ]; then
+        uadd=adduser
+        gadd=addgroup
+    fi
+    id -u $u >/dev/null || (${uadd} $u ; ${gadd} $u || true )>/dev/null
+    chown -R $u:$u ./
+}
+
+setup_helper() {
+    setup_user
+    for shared_file in config.log confdefs.h conftest.err; do
+        chmod a+rw $shared_file &>/dev/true || true
+    done
+    if [ "$distro_id" = "ID=fedora" ]; then
+        repo-install httpd
+    fi
+    if [ "$distro_id" = 'ID="opensuse-leap"' ]; then
+        repo-install apache2
+    fi
+    repo-install gcc g++ libcgroup-dev composer acl
+}
+
+multiple_compilers() {
+    # all of the tests ran before will have a subset of thos already
+    repo-install gcc g++ clang libcgroup-dev
+    export CC="$1"
+    export CXX="$2"
+    compiler_assertions "$1" "$2"
+    assert_line "checking for gcc... $1"
+    assert_line "checking how to run the C preprocessor... $1 -E"
+    assert_line "checking how to run the C++ preprocessor... $2 -E"
+    assert_line " * CPP.................: $1 -E"
+    assert_line " * CXX.................: $2"
+    compile_assertions_finished
+}
+
+check_docs () {
+    doc_files="domjudge-team-manual.pdf html/index.html team/domjudge-team-manual.pdf"
+    if [ -z "$1" ] || [ "$1" = "before" ]; then
+        for f in $doc_files; do
+            run ls "doc/manual/build/$f"
+            assert_failure
+        done
+    fi
+    if [ -z "$1" ]; then
+        run make docs
+    fi
+    if [ -z "$1" ] || [ "$1" = "after" ]; then
+        for f in $doc_files; do
+            run ls "doc/manual/build/$f"
+            assert_success
+        done
+    fi
+}
+
+build_default() {
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # In the current fedora container it fails with:
+        #/usr/bin/ld: cannot find -lstdc++: No such file or directory
+        #/usr/bin/ld: cannot find -lm: No such file or directory
+        #/usr/bin/ld: cannot find -lc: No such file or directory
+        # even though those libraries are installed and work on a normal
+        # fedora install.
+        skip
+    fi
+    user_make="$1"
+    user_install="$2"
+    if [ -z "$3" ]; then
+        prefix="/opt/domjudge"
+    else
+        prefix="$3"
+    fi
+    make_target="$4"
+    setup_helper
+    run run_configure --prefix="$prefix"
+    assert_line " * domserver...........: $prefix/domserver"
+    assert_regex "^ \* webserver group\.\.\.\.\.: (www-data|apache|nginx)$"
+    assert_line " * judgehost...........: $prefix/judgehost"
+    assert_line " * runguard group......: domjudge-run"
+    if [ -z "$make_target" ]; then
+        if [ "$user_make" = "root" ]; then
+            run make domserver
+        else
+            run su "$user_make" -c "make domserver"
+        fi
+        assert_success
+        if [ "$user_make" = "root" ]; then
+            run make judgehost
+        else
+            run su "$user_make" -c "make judgehost"
+        fi
+    else
+        if [ "$make_target" = "all" ]; then
+            check_docs "before"
+            run make "$make_target"
+            check_docs "after"
+        fi
+    fi
+    assert_success
+    if [ "$user_install" = "root" ]; then
+        run make install-domserver
+    else
+        run su "$user_install" -c "make install-domserver"
+    fi
+    assert_success
+    run ls "${prefix}"/domserver/webapp
+    assert_success
+    if [ "$user_install" = "root" ]; then
+        run make install-judgehost
+    else
+        run su "$user_install" -c "make install-judgehost"
+    fi
+    assert_success
+    run ls "${prefix}"/judgehost/judgings
+    assert_success
+}
+
+@test "make clean removes all generated files" {
+    docs_required_install &>/dev/zero
+    setup_helper
+    run run_configure
+    assert_success
+    run make all
+    if [ "$distro_id" = "ID=fedora" ]; then
+        # Because compiling fails
+        assert_failure
+    else
+        assert_success
+    fi
+    after_files=$(find . -type f)
+    # Find the new files not in the tarball
+    filtered_files=()
+    for af in $after_files; do
+        if [[ ! "${before_files[*]}" =~ "${af}" ]]; then
+            filtered_files+=($af)
+            echo "# Lost $af" >&3
+        fi
+    done
+    run make distclean
+    assert_success
+    after_clean_files=$(find . -type f)
+    # Files which were there to begin with should
+    # still be there.
+    for bf in $before_files; do
+        if [[ ! "${after_clean_files[*]}" =~ "${bf}" ]]; then
+            run echo "Wrongly removed: $bf, existed in original tarball but was removed after 'distclean'"
+            refute_line "Wrongly removed: $bf, existed in original tarball but was removed after 'distclean'"
+        fi
+    done
+    # Files which we now find should have been there in the beginning.
+    for af in $after_clean_files; do
+        if [[ ! "${before_files[*]}" =~ "${af}" ]]; then
+            run echo "Wrongly kept: $af, didn't exist in original tarball but was kept after 'distclean'"
+            refute_line "Wrongly kept: $af, didn't exist in original tarball but was kept after 'distclean'"
+        fi
+    done
+}
diff --git a/.github/jobs/configure-checks/setup_configure_image.sh b/.github/jobs/configure-checks/setup_configure_image.sh
index b71dad822c..9139e0652c 100755
--- a/.github/jobs/configure-checks/setup_configure_image.sh
+++ b/.github/jobs/configure-checks/setup_configure_image.sh
@@ -1,13 +1,25 @@
 #!/bin/bash
 
+. .github/jobs/ci_settings.sh
+
 set -euxo pipefail
 
-distro_id=$(grep "^ID=" /etc/os-release)
+ghtest="$1"
+export ghtest
+
+section_start "Install all tools"
+distro_id="$(grep "^ID=" /etc/os-release)"
 
 # Install everything for configure and testing
 case $distro_id in
+    "ID=arch")
+        pacman -Sy --noconfirm bats ;;
+    "ID=alpine")
+        apk add bash make bats tar ;;
     "ID=fedora")
         dnf install make bats util-linux -y ;;
+    'ID="opensuse-leap"')
+        zypper install -y make bats ;;
     *)
         apt-get update; apt-get full-upgrade -y
         apt-get install make bats -y ;;
@@ -15,6 +27,14 @@ esac
 
 # Install extra assert statements for bots
 cp submit/assert.bash .github/jobs/configure-checks/
+section_end
 
 # Run the configure tests for this usecase
-test_path="/__w/domjudge/domjudge/release" bats .github/jobs/configure-checks/all.bats
+test_path="/__w/domjudge/domjudge/release"
+export test_path
+
+if [ "${ghtest:?}" = "distclean" ]; then
+    .github/jobs/configure-checks/distclean.bats
+else
+    .github/jobs/configure-checks/all.bats
+fi
diff --git a/.github/workflows/autoconf-check.yml b/.github/workflows/autoconf-check.yml
index fa4fdcf707..cf2a456ba8 100644
--- a/.github/workflows/autoconf-check.yml
+++ b/.github/workflows/autoconf-check.yml
@@ -31,41 +31,54 @@ jobs:
           name: test-tarball
           path: ${{ github.workspace }}/release.tar.gz
 
-  redhat-family:
-    needs: build
-    strategy:
-      matrix:
-        version: [latest]
-        os: [fedora]
-    runs-on: ubuntu-latest
-    container:
-      image: ${{ matrix.os }}:${{ matrix.version }}
-    steps:
-      - name: Download Artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: test-tarball
-      - name: Unpack the "Release" tarball
-        run: tar xvf release.tar.gz
-      - name: Setup image and run bats tests
-        working-directory: release
-        run: .github/jobs/configure-checks/setup_configure_image.sh
-
   debian-family:
     needs: build
     strategy:
       matrix:
         version: [jammy, focal, rolling]
         os: [ubuntu]
+        ghtest: [gcc, g++, clang, libcgroup, distclean]
         releaseBranch:
           - ${{ contains(github.ref, 'gh-readonly-queue') }}
         exclude:
           - releaseBranch: false
         include:
+          # We seem to always need this duplication as we only want to run
+          # the ubuntu tests in the mergequeue and loop over multiple versions
+          # So: We can't bundle debian/ubuntu as we use the versions
+          # We can't put ubuntu in the other group as we can't add multiple
+          #   versions as include can't duplicate jobs so we could only add 1
+          #   ubuntu version.
+          - os: debian
+            version: stable
+            ghtest: gcc
+          - os: debian
+            version: stable
+            ghtest: g++
+          - os: debian
+            version: stable
+            ghtest: clang
+          - os: debian
+            version: stable
+            ghtest: libcgroup
           - os: debian
             version: stable
+            ghtest: distclean
           - os: debian
             version: testing
+            ghtest: gcc
+          - os: debian
+            version: testing
+            ghtest: g++
+          - os: debian
+            version: testing
+            ghtest: clang
+          - os: debian
+            version: testing
+            ghtest: libcgroup
+          - os: debian
+            version: testing
+            ghtest: distclean
     runs-on: ubuntu-latest
     env:
       DEBIAN_FRONTEND: noninteractive
@@ -81,5 +94,42 @@ jobs:
         run: tar xvf release.tar.gz
       - name: Setup image and run bats tests
         working-directory: release
-        run: .github/jobs/configure-checks/setup_configure_image.sh
+        run: |
+          echo "${{ matrix.ghtest }}"
+          .github/jobs/configure-checks/setup_configure_image.sh "${{ matrix.ghtest }}"
 
+  other-distros:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        # Alternatives alpine & arch are non-trivial
+        # alpine has issues with running bats,
+        # arch needs AUR for some dependencies (libcgroup)
+        # opensuse/leap shows errors with our g++ detection
+        os: [fedora]
+        ghtest: [gcc, g++, clang, libcgroup, distclean]
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.os }}:${{ matrix.version || 'latest' }}
+    steps:
+      - name: Download Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: test-tarball
+      - name: Install needed shell tools
+        run: |
+          distro_id=$(grep "^ID=" /etc/os-release)
+          # Install everything for configure and testing
+          case $distro_id in
+            "ID=fedora")
+              dnf install make bats -y ;;
+            *)
+              exit 10
+          esac
+      - name: Unpack the "Release" tarball
+        run: tar xvf release.tar.gz
+      - name: Setup image and run bats tests
+        working-directory: release
+        shell: bash
+        run: .github/jobs/configure-checks/setup_configure_image.sh "${{ matrix.ghtest }}"