.gitlab-ci.yml

# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# We define only two stages for development. The "build" stage only compiles the
# code and doesn't run any target code. The "test" stage runs the code compiled
# by the previous stage. Stages form dependencies in a block, all the build
# targets in one stage need to be completed before any of the next stage can
# start.
stages:
  - build
  - test
  - deploy

variables:
  # By default we don't fetch any third_party dependencies. Building requires to
  # fetch all the dependencies but many other workflows don't.
  GIT_SUBMODULE_STRATEGY: none
  # Set the default checkout directory to a fixed one so it is the same across
  # multiple jobs. cmake requires that build directory's realpath to not change.
  GIT_CLONE_PATH: $CI_BUILDS_DIR/libjxl

# A template for running in the generic cloud builders. These are tagged with
# "linux" and run on shared VMs.
.linux_host_template: &linux_host_template
  image: &jpegxl-builder gcr.io/jpegxl/jpegxl-builder@sha256:439a05c41f86a82067042fca16d5c9c7cc5836a8a5d8dba7fa0383f3c1e603c2
  tags:
    - linux
  # By default all the workflows run on master and on request. This can be
  # override by users of this template.
  only:
    - main
    - master
    - /^v[0-9]+\.[0-9]+\.x$/
    - tags
    - schedules
  # Retry on system failures. This can be typically caused by the runner or VM
  # being evicted.
  retry:
    max: 2
    when:
      - unknown_failure
      - api_failure
      - stuck_or_timeout_failure
      - runner_system_failure
  # Default variables. Note that adding a "variables:" section to a template
  # using this template will completely discard this section, not just add new
  # variables.
  variables:
    GIT_SUBMODULE_STRATEGY: none
    BUILD_DIR: build
    CC: clang-7
    CXX: clang++-7

# Common artifacts for the build stage
.default_build_paths: &default_build_paths
  - build/libjxl.so*
  - build/libjxl.dll
  # Binary tools like cjxl are "cjxl.exe" in windows builds. These
  # matches will match against either one.
  - build/lib/jxl_gbench*
  - build/tools/cjxl*
  - build/tools/djxl*
  - build/tools/benchmark_xl*
  - build/tools/decode_and_encode*
  - build/tools/comparison_viewer/compare_codecs*
  - build/tools/comparison_viewer/compare_images*
  - build/tools/viewer/viewer*
  # Brotli shared dependencies if built
  - build/third_party/brotli/libbrotli*so*
  - build/third_party/brotli/libbrotli*.dll
  # All the test binaries in the build/ directory and .cmake files used by
  # ctest, compressed into this file together to save space when uploading.
  - build/tests.tar.xz
  # The coverage data produced at compile time.
  - build/gcno.tar.xz
  # The packed html doxygen documentation, only produced by the coverage build.
  - build/htmldoc.tar.gz
  - build/stats.json
  - build/LICENSE*
  # The JNI library + wrapper + test.
  - build/tools/libjxl_jni*so*
  - build/tools/libjxl_jni*.dll
  - build/tools/jxl_jni_wrapper.jar
  - build/tools/jxl_jni_wrapper_test.jar

# Helper template to add the default artifacts paths to all build stages. If
# files in the "paths:" section are not present only a warning is generated.
.linux_host_build_template: &linux_host_build_template
  <<: *linux_host_template
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 week
    paths: *default_build_paths
  variables:
    # Building always requires to fetch all the repositories.
    GIT_SUBMODULE_STRATEGY: recursive
    BUILD_DIR: build
    CC: clang-7
    CXX: clang++-7

# Build from tarball source using README.md instructions.
build:x86_64:clang:tarball:
  <<: *linux_host_build_template
  stage: build
  variables:
    GIT_SUBMODULE_STRATEGY: none
    BUILD_DIR: build
    CC: clang-7
    CXX: clang++-7
  script:
    # Simulate a tarball checkout by removing .git and not fetching any
    # submodule.
    - mv .git .git-ignore || true
    - ./deps.sh
    - apt update
    - apt install -y cmake pkg-config libbrotli-dev
        libgif-dev libjpeg-dev libopenexr-dev libpng-dev libwebp-dev
    - mkdir build
    - cd build
    - cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF ..
    - cmake --build . -- -j

# linux x86_64 default Release mode.
build:x86_64:clang:release:
  <<: *linux_host_build_template
  stage: build
  script:
    # Check that the build files are up to date.
    - tools/build_cleaner.py
    - SKIP_TEST=1 PACK_TEST=1 STACK_SIZE=1
      ./ci.sh release -DCMAKE_INSTALL_PREFIX=`pwd`/prefix
    # Test that the package can be installed.
    - ninja -C build install
    - tools/build_stats.py --save build/stats.json
        cjxl djxl libjxl.so

test:x86_64:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:clang:release
  script:
    - ./ci.sh test
    # Quick run to make sure it doesn't crash. Not useful for actual benchmarks.
    - ./ci.sh gbench --benchmark_min_time=0

# x86_84 test coverage build
build:x86_64:clang:coverage:
  <<: *linux_host_build_template
  stage: build
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh coverage
    - tar -zcf build/htmldoc.tar.gz -C build/html .

test:x86_64:clang:coverage:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:clang:coverage
  script:
    - ./ci.sh test
    - ./ci.sh coverage_report
  variables:
    CC: clang-7
    CXX: clang++-7
    # Headers from submodules might be needed when generating the HTML reports.
    GIT_SUBMODULE_STRATEGY: recursive
    # Coverage builds use more stack for coverage tracking.
    TEST_STACK_LIMIT: 1024
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/coverage.txt
      - build/coverage*.html
      - build/coverage.xml
      - build/htmldoc.tar.gz

# Updates the "Pages" section in GitLab with the latest coverage report from
# master.
pages:
  <<: *linux_host_template
  stage: deploy
  only:
    - main
  dependencies:
    - test:x86_64:clang:coverage
  script:
    - mkdir public
    # Unpack the doxygen documentation as "pages".
    - tar -zxf build/htmldoc.tar.gz -C public
    # Link from the index to the coverage.html file.
    - sed -E "s,<!--header-->,<!--header--><a href=coverage.html>Coverage for ${CI_COMMIT_SHA}</a><br />," -i public/index.html
    # Copy the coverage files to public/coverage*
    - mv build/coverage* public/
    - sed -E "s,GCC Code Coverage Report,Coverage Report for <a href='${CI_PROJECT_URL}/commit/${CI_COMMIT_SHA}'>${CI_COMMIT_SHORT_SHA}</a>," -i public/coverage.html
  artifacts:
    paths:
      - public


# i686 (x86 32-bit) builders
build:i686:clang:release:
  <<: *linux_host_build_template
  stage: build
  script:
    - BUILD_TARGET=i686-linux-gnu SKIP_TEST=1 PACK_TEST=1 STACK_SIZE=1
        ./ci.sh release -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_ENABLE_FUZZERS=OFF
    - tools/build_stats.py --save build/stats.json
        --binutils=i686-linux-gnu-
        cjxl djxl libjxl.so

test:i686:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:i686:clang:release
  script:
    - ./ci.sh test

# Windows builders. These are cross-compiled from a linux host.
build:win64:clang:release:
  <<: *linux_host_build_template
  stage: build
  script:
    - BUILD_TARGET=x86_64-w64-mingw32
        SKIP_TEST=1 PACK_TEST=1
        ./ci.sh release -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_ENABLE_FUZZERS=OFF
          -DJPEGXL_ENABLE_PLUGINS=OFF

test:win64:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:win64:clang:release
  script:
    - BUILD_TARGET=x86_64-w64-mingw32 ./ci.sh test

# aarch64 release runs only on master and on request.
build:aarch64:clang:release:
  <<: *linux_host_build_template
  stage: build
  script:
    - BUILD_TARGET=aarch64-linux-gnu
        CMAKE_CXX_FLAGS="-DJXL_DISABLE_SLOW_TESTS" SKIP_TEST=1 PACK_TEST=1
        STACK_SIZE=1
        ./ci.sh release
    - tools/build_stats.py --save build/stats.json
        --binutils=aarch64-linux-gnu-
        cjxl djxl libjxl.so

test:aarch64:clang:release:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:aarch64:clang:release
  script:
    # LeakSanitizer doesn't work when running under qemu-arm.
    - ASAN_OPTIONS=detect_leaks=0 ./ci.sh test

build:aarch64:clang:release-nhp:
  <<: *linux_host_build_template
  stage: build
  script:
    - BUILD_TARGET=aarch64-linux-gnu
        CMAKE_CXX_FLAGS="-DJXL_DISABLE_SLOW_TESTS -DJXL_HIGH_PRECISION=0"
        SKIP_TEST=1 PACK_TEST=1 STACK_SIZE=1
        ./ci.sh release
    - tools/build_stats.py --save build/stats.json
        --binutils=aarch64-linux-gnu-
        cjxl djxl libjxl.so

test:aarch64:clang:release-nhp:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:aarch64:clang:release-nhp
  script:
    # LeakSanitizer doesn't work when running under qemu-arm.
    - ASAN_OPTIONS=detect_leaks=0 ./ci.sh test

# aarch64 asan build and test.
build:aarch64:clang:asan:
  <<: *linux_host_build_template
  stage: build
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh asan

test:aarch64:clang:asan:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:aarch64:clang:asan
  # Disable asan test on arm since they timeout.
  only:
    - tags
  script:
    - ./ci.sh test

build:x86_64:clang:tsan:
  <<: *linux_host_build_template
  stage: build
  script:
    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh tsan

test:x86_64:clang:tsan:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:x86_64:clang:tsan
  script:
    # tsan test runs fail with a small stack.
    - TEST_STACK_LIMIT=1024 ./ci.sh test

# Faster benchmark over a smaller set of images.
.benchmark_x86_64_template: &benchmark_x86_64_template
  <<: *linux_host_template
  stage: test
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/benchmark_results/

test:fast_benchmark:release:
  <<: *benchmark_x86_64_template
  dependencies:
    - build:x86_64:clang:release
  script:
    - STORE_IMAGES=0 ./ci.sh fast_benchmark

# This template runs on actual aarch64 hardware.
.benchmark_aarch64_template: &benchmark_aarch64_template
  <<: *linux_host_template
  image: gcr.io/jpegxl/jpegxl-builder-run-aarch64@sha256:27c2bb6319023ab94d66670135a971401869a3275a7e0dda177c9bb247d57e03
  stage: test
  tags:
    - aarch64
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/benchmark_results/
      - build/gbench.*

test:aarch64:fast_benchmark:release:
  <<: *benchmark_aarch64_template
  dependencies:
    - build:aarch64:clang:release
  script:
    # Bind to the big CPUs only.
    - ./ci.sh cpuset "${RUNNER_CPU_BIG}"
    - STORE_IMAGES=0 ./ci.sh fast_benchmark
    # Running the gbench in the big CPUs only.
    - ./ci.sh gbench

# Benchmark test that runs separately on the big and little CPUs of the runner.
test:aarch64:arm_benchmark:release:
  <<: *benchmark_aarch64_template
  dependencies:
    - build:aarch64:clang:release
  script:
    - ./ci.sh arm_benchmark

# Benchmark ToT on nightly builds. These are scheduled at midnight UTC.
test:benchmark:release:
  image: *jpegxl-builder
  stage: test
  only:
    - schedules
  variables:
    GIT_SUBMODULE_STRATEGY: none
  dependencies:
    - build:x86_64:clang:release
  script:
    - STORE_IMAGES=0 ./ci.sh benchmark
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    paths:
      - build/benchmark_results/

build:tidy:all:
  <<: *linux_host_build_template
  stage: build
  script:
    - CMAKE_BUILD_TYPE="Release" ./ci.sh tidy all
  allow_failure: true
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    expire_in: 1 year
    when: always
    paths:
      - build/clang-tidy.txt

# Emscripten (WASM) build.
build:ems:all:
  <<: *linux_host_build_template
  stage: build
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 SKIP_TEST=1 PACK_TEST=1 emconfigure ./ci.sh release

test:ems:all:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:ems:all
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 emconfigure ./ci.sh test

# Emscripten (WASM + SIMD) build.
build:ems_simd:all:
  <<: *linux_host_build_template
  stage: build
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 ENABLE_WASM_SIMD=1 SKIP_TEST=1 PACK_TEST=1 emconfigure ./ci.sh release

test:ems_simd:all:
  <<: *linux_host_template
  stage: test
  dependencies:
    - build:ems_simd:all
  script:
    - export V8=/opt/.jsvu/v8
    - source /opt/emsdk/emsdk_env.sh
    - BUILD_TARGET=wasm32 emconfigure ./ci.sh test