Refactor Evaluator Images, Pipeline "Run" type image specification

docs/docs/03-teachers-guide/02-task-configuration/03-tests.mdx

@@ -15,6 +15,14 @@ pipeline:
   - type: tests
 ```
 
+You can optionally specify a custom Docker image to run the tests in:
+
+```yaml
+pipeline:
+  - type: tests
+    image: kelvin/custom-image
+```
+
 :::
 
 Static tests can be defined by files in the task directory.
@@ -26,7 +34,7 @@ It is recommended to prepend the test number to each test because tests are orde
 
 This test will execute the student program and checks if it prints `2020` in the standard output.
 
-```
+```plaintext
 # 01_year.out
 2020
 ```
@@ -35,12 +43,12 @@ This test will execute the student program and checks if it prints `2020` in the
 
 The standard input is passed to the program and then the student's result on the output is compared to the expected stdout result.
 
-```
+```plaintext
 # 02_sum.in
 1 2 3 4
 ```
 
-```
+```plaintext
 # 02_sum.out
 10
 ```
@@ -49,7 +57,7 @@ The standard input is passed to the program and then the student's result on the
 
 Checks if the student's program created file `result.txt` with the expected content.
 
-```
+```plaintext
 # 03_nums.file_out.result.txt
 1 2 3 4 5 6 7 8 9 10
 ```
@@ -59,7 +67,7 @@ Checks if the student's program created file `result.txt` with the expected cont
 Provides the input file `data.txt` to student's program.
 It can be combined with stdout or file comparing.
 
-```
+```plaintext
 # 04_nums.file_in.data.txt
 1 2 3 4 5 6 7 8 9 10
 ```

docs/docs/03-teachers-guide/02-task-configuration/05-pipeline.mdx

@@ -196,6 +196,16 @@ pipeline:
 Commands prefixed with **#** are not shown on the result page.
 :::
 
+### i) Tests
+
+Action for running predefined input/output tests. See [Tests configuration](./03-tests.mdx) for more details.
+
+```yaml
+pipeline:
+  - type: tests
+    image: kelvin/run      # Docker image to use for running tests. Default: kelvin/run
+```
+
 ## Docker
 
 Own private actions can be implemented in any language in a [docker container](https://github.com/mrlvsb/kelvin/tree/master/evaluator/images) and published to the official docker hub.

evaluator/images/base/Dockerfile

@@ -1,30 +1,27 @@
 FROM ubuntu:24.04
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
+    build-essential \
     locales=2.39-0ubuntu8 \
     gcc=4:13.2.0-7ubuntu1 \
     g++=4:13.2.0-7ubuntu1 \
     gdb=15.0.50.20240403-0ubuntu1 \
     nasm=2.16.01-1build1 \
     python3=3.12.3-0ubuntu2.1 \
+    python3-pip=24.0+dfsg-1ubuntu1 \
+    python3-wheel=0.42.0-2 \
     cmake=3.28.3-1build7 && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    python3-pip && \
-    rm -rf /var/lib/apt/lists/*
-
-# For HTML sanitization
-RUN python3 -m pip install --break-system-packages bleach==5.0.1
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
 
-# Workaround for https://github.com/dotnet/sdk/issues/31457
-# It is included here, because it has to be present not only for building .NET projects,
-# but also for running them (e.g. in the `run` image).
-ENV DOTNET_EnableWriteXorExecute=0
+RUN python3 -m pip install --break-system-packages bleach==6.3.0

evaluator/images/cargo/Dockerfile

@@ -1,12 +1,16 @@
 FROM rust:1.90.0
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
     python3-pip && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # For HTML sanitization
-RUN python3 -m pip install --break-system-packages bleach==5.0.1
+RUN python3 -m pip install --break-system-packages bleach==6.3.0
 
 RUN rustup component add clippy
 

evaluator/images/cargo/entry.py

@@ -180,7 +180,7 @@ def run_cargo(command: str, args: List[str]) -> BuildResult:
     artifacts = output.binary_artifacts
     if len(artifacts) > 1:
         stdout += f"""
-Warning: multiple binary artifacts built ({', '.join([artifact.name for artifact in artifacts])}).
+Warning: multiple binary artifacts built ({", ".join([artifact.name for artifact in artifacts])}).
 Using the first one for further commands.
 """
     if len(artifacts) > 0:

evaluator/images/clang-tidy/Dockerfile

@@ -1,5 +1,15 @@
-FROM alpine:edge
-RUN apk update && apk add clang-extra-tools libc-dev linux-headers python3 py3-yaml libstdc++ g++
+FROM kelvin/gcc
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
+    clang-tidy \
+    python3-yaml && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
 ADD analyze.py build_urls.py /
 RUN /build_urls.py
 CMD /analyze.py

evaluator/images/dotnet/Dockerfile

@@ -1,19 +1,27 @@
 FROM kelvin/base
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    software-properties-common
+ENV DOTNET_EnableWriteXorExecute=0
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
+    software-properties-common && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN add-apt-repository ppa:dotnet/backports
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
     dotnet-sdk-9.0 \
-    aspnetcore-runtime-9.0 \
-    python3-pip && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN python3 -m pip install --break-system-packages bleach==5.0.1
+    aspnetcore-runtime-9.0 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ADD entry.py /
 CMD /entry.py

evaluator/images/java/Dockerfile

@@ -1,21 +1,29 @@
 FROM kelvin/base
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
     openjdk-21-jdk \
-    python3-pip \
     wget && \
+    apt-get clean && \
     rm -rf /var/lib/apt/lists/*
-#RUN wget https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz -P /tmp
+
 RUN wget https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.tar.gz -P /tmp
+
 RUN tar xf /tmp/apache-maven-*-bin.tar.gz -C /opt
+
 RUN ln -s /opt/apache-maven-3.9.9/ /opt/maven
-RUN echo 'export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))\n\
-export M2_HOME=/opt/maven\n\
-export MAVEN_HOME=/opt/maven\n\
-export PATH=$M2_HOME/bin:$PATH' > /etc/profile.d/maven.sh
-RUN chmod +x /etc/profile.d/maven.sh
-RUN /usr/bin/python3 -m pip install --break-system-packages bleach==5.0.1
+
+# To find out exactly folder path, run the docker image and enter
+# `update-alternatives --config java` or `dirname $(dirname $(readlink -f $(which java)))` command
+# We support only amd64 architecture, so the path is hardcoded to java-21-openjdk-amd64, but it can be changed if needed
+ENV JAVA_HOME="/usr/lib/jvm/java-21-openjdk-amd64"
+ENV M2_HOME="/opt/maven"
+ENV MAVEN_HOME="/opt/maven"
+ENV PATH="$M2_HOME/bin:$PATH"
+
 RUN mkdir /.m2
 RUN chmod u+rwx,g+rwx,o+rwx /.m2
 

evaluator/images/java/entry.py

@@ -126,10 +126,6 @@ def build_java_project(run_tests: bool) -> BuildResult:
     if len(executable_class_names) == 1:
         main_script_lines = [
             "#!/bin/bash",
-            f"JAVA_HOME={os.environ['JAVA_HOME']}",
-            f"MAVEN_HOME={os.environ['MAVEN_HOME']}",
-            f"M2_HOME={os.environ['M2_HOME']}",
-            f"PATH={os.environ['PATH']}",
             f"mvn --quiet exec:java -Dexec.mainClass={executable_class_names[0]}",
         ]
         script_name = "main"
@@ -170,25 +166,6 @@ def build_java_project(run_tests: bool) -> BuildResult:
     )
 
 
-def get_java_home():
-    try:
-        java_bin_path = subprocess.check_output("which java", shell=True, text=True).strip()
-        java_real_path = subprocess.check_output(
-            f"readlink -f {java_bin_path}", shell=True, text=True
-        ).strip()
-        java_home = os.path.dirname(os.path.dirname(java_real_path))
-        return java_home
-    except subprocess.CalledProcessError:
-        # Zde můžete logovat chybu nebo vrátit výchozí hodnotu
-        return None
-
-
-# set environment variables
-os.environ["JAVA_HOME"] = get_java_home() or "/usr/lib/jvm/default-java"
-os.environ["M2_HOME"] = "/opt/maven"
-os.environ["MAVEN_HOME"] = "/opt/maven"
-os.environ["PATH"] = f"{os.environ['M2_HOME']}/bin:{os.environ['PATH']}"
-
 run_tests = os.getenv("PIPE_UNITTESTS", False)
 result = build_java_project(run_tests)
 

evaluator/images/pythonrun/Dockerfile

@@ -1,6 +1,3 @@
-FROM kelvin/run
-RUN apt-get update &&  \
-    apt-get install -y --no-install-recommends python3-pip python3-wheel && \
-    rm -rf /var/lib/apt/lists/*
+FROM kelvin/base
 
-RUN pip3 install --break-system-packages pytest
+RUN pip3 install --break-system-packages pytest==9.0.2 flake8==7.3.0

evaluator/images/run/Dockerfile

@@ -1,9 +1,16 @@
 FROM kelvin/base
 
-RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y asciinema imagemagick webp python3-magic \
-    openjdk-21-jdk && \
-    rm -rf /var/lib/apt/lists/*
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y \
+    -o APT::Install-Recommends=false \
+    -o APT::Install-Suggests=false \
+    asciinema \
+    imagemagick \
+    webp \
+    python3-magic && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ADD entry.py /
 CMD /entry.py

evaluator/pipelines.py

@@ -86,7 +86,7 @@ def fmt_value(v):
         "-v",
         evaluation.submit_path + ":/work",
         "--ulimit",
-        f'fsize={limits["fsize"]}:{limits["fsize"]}',
+        f"fsize={limits['fsize']}:{limits['fsize']}",
         "-m",
         str(limits["memory"]),
         "--memory-swap",
@@ -267,19 +267,28 @@ def to_file(input):
 
 
 class TestsPipe:
-    def __init__(self, executable="./main", limits=None, timeout=5, before=None, **kwargs):
+    def __init__(
+        self,
+        executable="./main",
+        limits=None,
+        timeout=5,
+        before=None,
+        image="kelvin/run",
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self.executable = [executable] if isinstance(executable, str) else executable
         self.limits = limits
         self.timeout = timeout
         self.before = [] if not before else before
+        self.image = image
 
     def run(self, evaluation):
         results = []
         result_dir = os.path.join(evaluation.result_path, self.id)
         os.mkdir(result_dir)
 
-        image = prepare_container(docker_image("kelvin/run"), self.before)
+        image = prepare_container(docker_image(self.image), self.before)
         container = (
             subprocess.check_output(
                 create_docker_cmd(

evaluator/testsets.py

@@ -233,7 +233,7 @@ def parse_conf_pipeline(self, conf):
 
                     self.pipeline.append(pipe)
                 except Exception as e:
-                    self.add_warning(f'pipe {item["type"]}: {e}\n{traceback.format_exc()}')
+                    self.add_warning(f"pipe {item['type']}: {e}\n{traceback.format_exc()}")
 
     def parse_conf_tests(self, conf):
         allowed_keys = ["name", "title", "exit_code", "args", "files"]

frontend/src/PipelineValidation.js

@@ -567,7 +567,11 @@ const rules = new DictRule({
         ],
         tests: [
             new DockerPipeRule({
-                executable: new UnionRule(new ValueRule(), new ArrayRule())
+                executable: new UnionRule(new ValueRule(), new ArrayRule()),
+                image: [
+                    new ValueRule(),
+                    'Docker image to use for running tests. Default: <strong>kelvin/run</strong>'
+                ]
             }),
             'Run input/output/files tests on compiled program.'
         ],