diff --git a/docs/docs/03-teachers-guide/02-task-configuration/03-tests.mdx b/docs/docs/03-teachers-guide/02-task-configuration/03-tests.mdx index 920fe61ca..b6664d5e5 100644 --- a/docs/docs/03-teachers-guide/02-task-configuration/03-tests.mdx +++ b/docs/docs/03-teachers-guide/02-task-configuration/03-tests.mdx @@ -15,6 +15,14 @@ pipeline: - type: tests ``` +You can optionally specify a custom Docker image to run the tests in: + +```yaml +pipeline: + - type: tests + image: kelvin/custom-image +``` + ::: Static tests can be defined by files in the task directory. @@ -26,7 +34,7 @@ It is recommended to prepend the test number to each test because tests are orde This test will execute the student program and checks if it prints `2020` in the standard output. -``` +```plaintext # 01_year.out 2020 ``` @@ -35,12 +43,12 @@ This test will execute the student program and checks if it prints `2020` in the The standard input is passed to the program and then the student's result on the output is compared to the expected stdout result. -``` +```plaintext # 02_sum.in 1 2 3 4 ``` -``` +```plaintext # 02_sum.out 10 ``` @@ -49,7 +57,7 @@ The standard input is passed to the program and then the student's result on the Checks if the student's program created file `result.txt` with the expected content. -``` +```plaintext # 03_nums.file_out.result.txt 1 2 3 4 5 6 7 8 9 10 ``` @@ -59,7 +67,7 @@ Checks if the student's program created file `result.txt` with the expected cont Provides the input file `data.txt` to student's program. It can be combined with stdout or file comparing. -``` +```plaintext # 04_nums.file_in.data.txt 1 2 3 4 5 6 7 8 9 10 ``` diff --git a/docs/docs/03-teachers-guide/02-task-configuration/05-pipeline.mdx b/docs/docs/03-teachers-guide/02-task-configuration/05-pipeline.mdx index 42b5ac8df..e15d5450e 100644 --- a/docs/docs/03-teachers-guide/02-task-configuration/05-pipeline.mdx +++ b/docs/docs/03-teachers-guide/02-task-configuration/05-pipeline.mdx @@ -196,6 +196,16 @@ pipeline: Commands prefixed with **#** are not shown on the result page. ::: +### i) Tests + +Action for running predefined input/output tests. See [Tests configuration](./03-tests.mdx) for more details. + +```yaml +pipeline: + - type: tests + image: kelvin/run # Docker image to use for running tests. Default: kelvin/run +``` + ## Docker Own private actions can be implemented in any language in a [docker container](https://github.com/mrlvsb/kelvin/tree/master/evaluator/images) and published to the official docker hub. diff --git a/evaluator/images/base/Dockerfile b/evaluator/images/base/Dockerfile index cc5eb6011..6faa41b43 100644 --- a/evaluator/images/base/Dockerfile +++ b/evaluator/images/base/Dockerfile @@ -1,30 +1,27 @@ FROM ubuntu:24.04 -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + -o APT::Install-Recommends=false \ + -o APT::Install-Suggests=false \ + build-essential \ locales=2.39-0ubuntu8 \ gcc=4:13.2.0-7ubuntu1 \ g++=4:13.2.0-7ubuntu1 \ gdb=15.0.50.20240403-0ubuntu1 \ nasm=2.16.01-1build1 \ python3=3.12.3-0ubuntu2.1 \ + python3-pip=24.0+dfsg-1ubuntu1 \ + python3-wheel=0.42.0-2 \ cmake=3.28.3-1build7 && \ - rm -rf /var/lib/apt/lists/* - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - python3-pip && \ - rm -rf /var/lib/apt/lists/* - -# For HTML sanitization -RUN python3 -m pip install --break-system-packages bleach==5.0.1 + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 -# Workaround for https://github.com/dotnet/sdk/issues/31457 -# It is included here, because it has to be present not only for building .NET projects, -# but also for running them (e.g. in the `run` image). -ENV DOTNET_EnableWriteXorExecute=0 +RUN python3 -m pip install --break-system-packages bleach==6.3.0 diff --git a/evaluator/images/cargo/Dockerfile b/evaluator/images/cargo/Dockerfile index 965a6e3b5..df8fab9ca 100644 --- a/evaluator/images/cargo/Dockerfile +++ b/evaluator/images/cargo/Dockerfile @@ -1,12 +1,16 @@ FROM rust:1.90.0 -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + -o APT::Install-Recommends=false \ + -o APT::Install-Suggests=false \ python3-pip && \ - rm -rf /var/lib/apt/lists/* + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # For HTML sanitization -RUN python3 -m pip install --break-system-packages bleach==5.0.1 +RUN python3 -m pip install --break-system-packages bleach==6.3.0 RUN rustup component add clippy diff --git a/evaluator/images/cargo/entry.py b/evaluator/images/cargo/entry.py index 3590d8fff..4aafd226b 100755 --- a/evaluator/images/cargo/entry.py +++ b/evaluator/images/cargo/entry.py @@ -180,7 +180,7 @@ def run_cargo(command: str, args: List[str]) -> BuildResult: artifacts = output.binary_artifacts if len(artifacts) > 1: stdout += f""" -Warning: multiple binary artifacts built ({', '.join([artifact.name for artifact in artifacts])}). +Warning: multiple binary artifacts built ({", ".join([artifact.name for artifact in artifacts])}). Using the first one for further commands. """ if len(artifacts) > 0: diff --git a/evaluator/images/clang-tidy/Dockerfile b/evaluator/images/clang-tidy/Dockerfile index 50deca9d0..9b6745183 100644 --- a/evaluator/images/clang-tidy/Dockerfile +++ b/evaluator/images/clang-tidy/Dockerfile @@ -1,5 +1,15 @@ -FROM alpine:edge -RUN apk update && apk add clang-extra-tools libc-dev linux-headers python3 py3-yaml libstdc++ g++ +FROM kelvin/gcc + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + -o APT::Install-Recommends=false \ + -o APT::Install-Suggests=false \ + clang-tidy \ + python3-yaml && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + ADD analyze.py build_urls.py / RUN /build_urls.py CMD /analyze.py diff --git a/evaluator/images/dotnet/Dockerfile b/evaluator/images/dotnet/Dockerfile index fa741ff63..576ecc9d6 100644 --- a/evaluator/images/dotnet/Dockerfile +++ b/evaluator/images/dotnet/Dockerfile @@ -1,19 +1,27 @@ FROM kelvin/base -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - software-properties-common +ENV DOTNET_EnableWriteXorExecute=0 + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + -o APT::Install-Recommends=false \ + -o APT::Install-Suggests=false \ + software-properties-common && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN add-apt-repository ppa:dotnet/backports -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + -o APT::Install-Recommends=false \ + -o APT::Install-Suggests=false \ dotnet-sdk-9.0 \ - aspnetcore-runtime-9.0 \ - python3-pip && \ - rm -rf /var/lib/apt/lists/* - -RUN python3 -m pip install --break-system-packages bleach==5.0.1 + aspnetcore-runtime-9.0 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ADD entry.py / CMD /entry.py diff --git a/evaluator/images/java/Dockerfile b/evaluator/images/java/Dockerfile index 97295493e..c4aa97aef 100644 --- a/evaluator/images/java/Dockerfile +++ b/evaluator/images/java/Dockerfile @@ -1,21 +1,29 @@ FROM kelvin/base -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + -o APT::Install-Recommends=false \ + -o APT::Install-Suggests=false \ openjdk-21-jdk \ - python3-pip \ wget && \ + apt-get clean && \ rm -rf /var/lib/apt/lists/* -#RUN wget https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz -P /tmp + RUN wget https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.tar.gz -P /tmp + RUN tar xf /tmp/apache-maven-*-bin.tar.gz -C /opt + RUN ln -s /opt/apache-maven-3.9.9/ /opt/maven -RUN echo 'export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))\n\ -export M2_HOME=/opt/maven\n\ -export MAVEN_HOME=/opt/maven\n\ -export PATH=$M2_HOME/bin:$PATH' > /etc/profile.d/maven.sh -RUN chmod +x /etc/profile.d/maven.sh -RUN /usr/bin/python3 -m pip install --break-system-packages bleach==5.0.1 + +# To find out exactly folder path, run the docker image and enter +# `update-alternatives --config java` or `dirname $(dirname $(readlink -f $(which java)))` command +# We support only amd64 architecture, so the path is hardcoded to java-21-openjdk-amd64, but it can be changed if needed +ENV JAVA_HOME="/usr/lib/jvm/java-21-openjdk-amd64" +ENV M2_HOME="/opt/maven" +ENV MAVEN_HOME="/opt/maven" +ENV PATH="$M2_HOME/bin:$PATH" + RUN mkdir /.m2 RUN chmod u+rwx,g+rwx,o+rwx /.m2 diff --git a/evaluator/images/java/entry.py b/evaluator/images/java/entry.py index 6638fd33c..26007d9a9 100755 --- a/evaluator/images/java/entry.py +++ b/evaluator/images/java/entry.py @@ -126,10 +126,6 @@ def build_java_project(run_tests: bool) -> BuildResult: if len(executable_class_names) == 1: main_script_lines = [ "#!/bin/bash", - f"JAVA_HOME={os.environ['JAVA_HOME']}", - f"MAVEN_HOME={os.environ['MAVEN_HOME']}", - f"M2_HOME={os.environ['M2_HOME']}", - f"PATH={os.environ['PATH']}", f"mvn --quiet exec:java -Dexec.mainClass={executable_class_names[0]}", ] script_name = "main" @@ -170,25 +166,6 @@ def build_java_project(run_tests: bool) -> BuildResult: ) -def get_java_home(): - try: - java_bin_path = subprocess.check_output("which java", shell=True, text=True).strip() - java_real_path = subprocess.check_output( - f"readlink -f {java_bin_path}", shell=True, text=True - ).strip() - java_home = os.path.dirname(os.path.dirname(java_real_path)) - return java_home - except subprocess.CalledProcessError: - # Zde můžete logovat chybu nebo vrátit výchozí hodnotu - return None - - -# set environment variables -os.environ["JAVA_HOME"] = get_java_home() or "/usr/lib/jvm/default-java" -os.environ["M2_HOME"] = "/opt/maven" -os.environ["MAVEN_HOME"] = "/opt/maven" -os.environ["PATH"] = f"{os.environ['M2_HOME']}/bin:{os.environ['PATH']}" - run_tests = os.getenv("PIPE_UNITTESTS", False) result = build_java_project(run_tests) diff --git a/evaluator/images/pythonrun/Dockerfile b/evaluator/images/pythonrun/Dockerfile index 910f6a45c..8e0a8a1d8 100644 --- a/evaluator/images/pythonrun/Dockerfile +++ b/evaluator/images/pythonrun/Dockerfile @@ -1,6 +1,3 @@ -FROM kelvin/run -RUN apt-get update && \ - apt-get install -y --no-install-recommends python3-pip python3-wheel && \ - rm -rf /var/lib/apt/lists/* +FROM kelvin/base -RUN pip3 install --break-system-packages pytest +RUN pip3 install --break-system-packages pytest==9.0.2 flake8==7.3.0 diff --git a/evaluator/images/run/Dockerfile b/evaluator/images/run/Dockerfile index 923fe4f31..f5f4f3306 100644 --- a/evaluator/images/run/Dockerfile +++ b/evaluator/images/run/Dockerfile @@ -1,9 +1,16 @@ FROM kelvin/base -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y asciinema imagemagick webp python3-magic \ - openjdk-21-jdk && \ - rm -rf /var/lib/apt/lists/* +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + -o APT::Install-Recommends=false \ + -o APT::Install-Suggests=false \ + asciinema \ + imagemagick \ + webp \ + python3-magic && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ADD entry.py / CMD /entry.py diff --git a/evaluator/pipelines.py b/evaluator/pipelines.py index e9cc24603..e43f01d50 100644 --- a/evaluator/pipelines.py +++ b/evaluator/pipelines.py @@ -86,7 +86,7 @@ def fmt_value(v): "-v", evaluation.submit_path + ":/work", "--ulimit", - f'fsize={limits["fsize"]}:{limits["fsize"]}', + f"fsize={limits['fsize']}:{limits['fsize']}", "-m", str(limits["memory"]), "--memory-swap", @@ -267,19 +267,28 @@ def to_file(input): class TestsPipe: - def __init__(self, executable="./main", limits=None, timeout=5, before=None, **kwargs): + def __init__( + self, + executable="./main", + limits=None, + timeout=5, + before=None, + image="kelvin/run", + **kwargs, + ): super().__init__(**kwargs) self.executable = [executable] if isinstance(executable, str) else executable self.limits = limits self.timeout = timeout self.before = [] if not before else before + self.image = image def run(self, evaluation): results = [] result_dir = os.path.join(evaluation.result_path, self.id) os.mkdir(result_dir) - image = prepare_container(docker_image("kelvin/run"), self.before) + image = prepare_container(docker_image(self.image), self.before) container = ( subprocess.check_output( create_docker_cmd( diff --git a/evaluator/testsets.py b/evaluator/testsets.py index ca31f3823..5763851f7 100644 --- a/evaluator/testsets.py +++ b/evaluator/testsets.py @@ -233,7 +233,7 @@ def parse_conf_pipeline(self, conf): self.pipeline.append(pipe) except Exception as e: - self.add_warning(f'pipe {item["type"]}: {e}\n{traceback.format_exc()}') + self.add_warning(f"pipe {item['type']}: {e}\n{traceback.format_exc()}") def parse_conf_tests(self, conf): allowed_keys = ["name", "title", "exit_code", "args", "files"] diff --git a/frontend/src/PipelineValidation.js b/frontend/src/PipelineValidation.js index be666fa9f..67e6f55cc 100644 --- a/frontend/src/PipelineValidation.js +++ b/frontend/src/PipelineValidation.js @@ -567,7 +567,11 @@ const rules = new DictRule({ ], tests: [ new DockerPipeRule({ - executable: new UnionRule(new ValueRule(), new ArrayRule()) + executable: new UnionRule(new ValueRule(), new ArrayRule()), + image: [ + new ValueRule(), + 'Docker image to use for running tests. Default: kelvin/run' + ] }), 'Run input/output/files tests on compiled program.' ],