.gitlab-ci.yml

stages:
  - static_analysis
  - test
  - long_tests
  - perf
  - cleanup

# 1. Git strategy disabled as shown at:
#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-strategy
# 2. Git clean flags disabled as shown at:
#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-clean-flags
variables:
  GITLAB_MAKE: "make -f .gitlab.mk"
  GIT_STRATEGY: none
  GIT_CLEAN_FLAGS: none
  OSX_VARDIR: /tmp/tnt

# 1. Git checkout strategy used from:
#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-checkout
# 2. Submodule strategy used from:
#    https://docs.gitlab.com/ee/ci/yaml/README.html#git-submodule-strategy
# List of steps:
# 1) Check/clone the Tarantool repository with submodules.
# 2) For shell based jobs change ownership of all the sources
#    to gitlab-runner user (NOTE: in Docker based jobs the
#    gitlab-runner user is not known).
# 3) Fetch Tarantool sources with branches and force checkout
#    of the testing commit.
# 4) Drop a tag that points to a current commit (if any) on a job triggered by
#    pushing to a branch (as against of pushing a tag). Otherwise we may get
#    two jobs for the same x.y.z-0-gxxxxxxxxx build: one is run by pushing a
#    branch and another by pushing a tag. The idea is to hide the new tag from
#    the branch job as if a tag would be pushed strictly after all branch jobs
#    for the same commit. See gh-3745.
# 5) Update submodules recursively (use force where supports).
# 6) Cleanup all the sources from all files except from repository.
before_script:
  - /bin/bash -c "pwd && ls -d .git || git clone --recurse-submodules ${CI_REPOSITORY_URL} ."
  - /bin/bash -c "! docker -v || (uid=`id -u` ; gid=`id -g` ; docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c \"chown -R \$uid:\$gid * .[^.]*\")"
  - /bin/bash -c "git fetch -p && git checkout -f ${CI_COMMIT_SHORT_SHA}"
  - /bin/bash -c "[ -z \"${CI_COMMIT_TAG:-}\" ] && git tag -d \"$(git tag --points-at HEAD)\" 2>/dev/null || true"
  - /bin/bash -c "git submodule sync --recursive && git submodule update --force --init --recursive || git submodule update --recursive --init"
  - /bin/bash -c "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status"

# Jobs templates

.artifacts_files_template: &artifacts_files_definition
  artifacts:
    expire_in: 1000 yrs
    when: always
    paths:
      - "artifacts"
  after_script:
    - cp -r test/var/artifacts .

.deploy_only_template: &deploy_only_definition
  only:
    - master
    - tags
  except:
    - schedules
    - external_pull_requests
    - merge_requests

.pack_only_template: &pack_only_definition
  except:
    - master
    - tags

.perf_only_template: &perf_only_definition
  only:
    - master
    - tags
    - /^.*-perf$/
  except:
    - schedules
  variables: &perf_vars_definition
    IMAGE_PERF: "${CI_REGISTRY}/${CI_PROJECT_PATH}/perf/ubuntu-bionic:perf_master"
    IMAGE_PERF_BUILT: "${CI_REGISTRY}/${CI_PROJECT_PATH}/perf_tmp/ubuntu-bionic:perf_${CI_COMMIT_SHORT_SHA}"

.docker_test_template: &docker_test_definition
  <<: *artifacts_files_definition
  image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-stretch:latest"
  stage: test
  tags:
    - docker_test

.docker_test_clang11_template: &docker_test_clang8_definition
  <<: *artifacts_files_definition
  image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-buster:latest"
  stage: test
  tags:
    - docker_test

.pack_artifacts_files_template: &pack_artifacts_files_definition
  <<: *artifacts_files_definition
  after_script:
    - cp -r build/usr/src/*/tarantool-*/test/var/artifacts .

.pack_template: &pack_definition
  <<: *pack_only_definition
  stage: test
  tags:
    - deploy
  script:
    - ${GITLAB_MAKE} package

.pack_test_template: &pack_test_definition
  <<: *pack_only_definition
  <<: *pack_artifacts_files_definition
  stage: test
  tags:
    - deploy_test
  script:
    - ${GITLAB_MAKE} package

.deploy_template: &deploy_definition
  <<: *deploy_only_definition
  stage: test
  tags:
    - deploy
  script:
    - ${GITLAB_MAKE} deploy

.deploy_test_template: &deploy_test_definition
  <<: *deploy_only_definition
  <<: *pack_artifacts_files_definition
  stage: test
  tags:
    - deploy_test
  script:
    - ${GITLAB_MAKE} deploy

.osx_template: &osx_definition
  <<: *artifacts_files_definition
  stage: test
  script:
    - ${GITLAB_MAKE} test_osx
  after_script:
    # Artifacts can't be used from outside the project directory, check:
    #   https://docs.gitlab.com/ee/ci/yaml/README.html#artifactspaths
    # "Paths are relative to the project directory ($CI_PROJECT_DIR)"
    # "and can’t directly link outside it."
    - cp -rf ${OSX_VARDIR}/artifacts .

.vbox_template: &vbox_definition
  <<: *artifacts_files_definition
  stage: test
  after_script:
    - >
      scp -r -P ${VMS_PORT} ${VMS_USER}@127.0.0.1:tarantool/test/var/artifacts . ;
      ${GITLAB_MAKE} vms_shutdown

.perf_docker_test_template: &perf_docker_test_definition
  <<: *perf_only_definition
  image: ${IMAGE_PERF_BUILT}
  stage: perf
  artifacts:
    when: always
    paths:
      - "*_result.txt"
      - "*_t_version.txt"
  script:
    - ${GITLAB_MAKE} perf_run

.perf_cleanup_definition: &perf_cleanup_definition
  <<: *perf_only_definition
  stage: cleanup
  script:
    - ${GITLAB_MAKE} perf_cleanup

.jepsen_test_definition: &jepsen_test_definition
  <<: *docker_test_definition
  script:
    - ${GITLAB_MAKE} test_jepsen
  stage: long_tests
  when: manual
  tags:
    - mcs_jepsen_docker
  artifacts:
    paths:
      - jepsen-tests-prefix/src/jepsen-tests/store
    expire_in: 6 month

# Static Analysis

luacheck:
  <<: *docker_test_definition
  stage: static_analysis
  tags:
    - deploy_test
  script:
    - ${GITLAB_MAKE} test_debian_docker_luacheck

# Tests

out_of_source:
  stage: test
  tags:
    - deploy_test
  script:
    - ${GITLAB_MAKE} test_oos_build

release:
  <<: *docker_test_definition
  script:
    - ${GITLAB_MAKE} test_debian_no_deps

debug:
  <<: *docker_test_definition
  script:
    - ${GITLAB_MAKE} test_coverage_debian_no_deps

release_clang:
  <<: *docker_test_definition
  variables:
    CC: clang
    CXX: clang++
  script:
    - ${GITLAB_MAKE} test_debian_no_deps

release_lto:
  <<: *docker_test_clang8_definition
  variables:
    CMAKE_EXTRA_PARAMS: -DENABLE_LTO=ON
  script:
    - ${GITLAB_MAKE} test_debian_no_deps

release_lto_clang11:
  <<: *docker_test_clang8_definition
  variables:
    CC: clang-11
    CXX: clang++-11
    CMAKE_EXTRA_PARAMS: -DENABLE_LTO=ON
  script:
    - ${GITLAB_MAKE} test_debian_no_deps

release_asan_clang11:
  <<: *docker_test_clang8_definition
  script:
    - ${GITLAB_MAKE} test_asan_debian_no_deps

osx_13_release:
  tags:
    - osx_13
  variables:
    MACOSX_DEPLOYMENT_TARGET: '10.13'
  <<: *osx_definition

osx_14_release:
  tags:
    - osx_14
  <<: *osx_definition

osx_15_release:
  tags:
    - osx_15
  <<: *osx_definition

osx_15_release_lto:
  tags:
    - osx_15
  variables:
    CMAKE_EXTRA_PARAMS: -DENABLE_LTO=ON
  <<: *osx_definition

freebsd_12_release:
  <<: *vbox_definition
  tags:
    - vms_freebsd_12
  variables:
    VMS_NAME: 'freebsd_12'
    VMS_USER: 'vagrant'
    VMS_PORT: '2232'
    MAKE: 'gmake'
  script:
    - ${GITLAB_MAKE} vms_start
    - ${GITLAB_MAKE} vms_test_freebsd

jepsen-single-instance:
  <<: *jepsen_test_definition
  variables:
    LEIN_OPT: '--nemesis standard'

jepsen-single-instance-txm:
  <<: *jepsen_test_definition
  variables:
    LEIN_OPT: '--nemesis standard --mvcc'

jepsen-cluster:
  <<: *jepsen_test_definition
  variables:
    LEIN_OPT: '--nemesis standard'
    INSTANCE_COUNT: '5'

jepsen-cluster-txm:
  <<: *jepsen_test_definition
  variables:
    LEIN_OPT: '--nemesis standard --mvcc'
    INSTANCE_COUNT: '5'

default_gcc_centos_7:
  <<: *pack_test_definition
  variables:
    PACKPACK_EXTRA_DOCKER_RUN_PARAMS: '-e CC=/usr/bin/gcc -e CXX=/usr/bin/g++'
    OS: 'el'
    DIST: '7'

# ####
# Perf
# ####

# Pre-testing part

perf_bootstrap:
  <<: *perf_only_definition
  stage: test
  tags:
    - deploy
  script:
    - ${GITLAB_MAKE} perf_prepare
  after_script:
    - ${GITLAB_MAKE} perf_cleanup_image

# Testing part

perf_tpch:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh2_perf
  image: ${IMAGE_PERF_BUILT}_tpch
  variables:
    <<: *perf_vars_definition
    BENCH: 'tpch'
  artifacts:
    when: always
    paths:
      - "bench-*.csv"
      - "*_t_version.txt"
      - "*_result.txt"

perf_sysbench:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh3_perf
  variables:
    <<: *perf_vars_definition
    BENCH: 'sysbench'

perf_tpcc:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh3_perf
  variables:
    <<: *perf_vars_definition
    BENCH: 'tpcc'

perf_ycsb_hash:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh2_perf
  variables:
    <<: *perf_vars_definition
    BENCH: 'ycsb'
    ARG: 'hash'

perf_ycsb_tree:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh2_perf
  variables:
    <<: *perf_vars_definition
    BENCH: 'ycsb'
    ARG: 'tree'

perf_nosqlbench_hash:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh1_perf
  variables:
    <<: *perf_vars_definition
    BENCH: 'nosqlbench'
    ARG: 'hash'

perf_nosqlbench_tree:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh1_perf
  variables:
    <<: *perf_vars_definition
    BENCH: 'nosqlbench'
    ARG: 'tree'

perf_cbench:
  <<: *perf_docker_test_definition
  tags:
    - docker_sh2_perf
  variables:
    <<: *perf_vars_definition
    BENCH: 'cbench'

perf_linkbench_ssd:
  <<: *perf_docker_test_definition
  tags:
    - docker_perf_ssd
  variables:
    <<: *perf_vars_definition
    BENCH: 'linkbench'

# Post-testing part

remove_images_sh1:
  <<: *perf_cleanup_definition
  tags:
    - sh1_shell

remove_images_sh2:
  <<: *perf_cleanup_definition
  tags:
    - sh2_shell

remove_images_sh3:
  <<: *perf_cleanup_definition
  tags:
    - sh3_shell

remove_images_sh9:
  <<: *perf_cleanup_definition
  tags:
    - sh9_shell

# Packages and sources

sources:
  <<: *pack_definition
  script:
    - ${GITLAB_MAKE} source

centos_6:
  <<: *pack_definition
  variables:
    OS: 'el'
    DIST: '6'

centos_7:
  <<: *pack_test_definition
  variables:
    OS: 'el'
    DIST: '7'

centos_8:
  <<: *pack_test_definition
  variables:
    OS: 'el'
    DIST: '8'

fedora_28:
  <<: *pack_test_definition
  variables:
    OS: 'fedora'
    DIST: '28'

fedora_29:
  <<: *pack_test_definition
  variables:
    OS: 'fedora'
    DIST: '29'

fedora_30:
  <<: *pack_test_definition
  variables:
    OS: 'fedora'
    DIST: '30'

fedora_31:
  <<: *pack_test_definition
  variables:
    OS: 'fedora'
    DIST: '31'

ubuntu_14_04:
  <<: *pack_definition
  variables:
    OS: 'ubuntu'
    DIST: 'trusty'

ubuntu_16_04:
  <<: *pack_definition
  variables:
    OS: 'ubuntu'
    DIST: 'xenial'

ubuntu_18_04:
  <<: *pack_definition
  variables:
    OS: 'ubuntu'
    DIST: 'bionic'

ubuntu_19_10:
  <<: *pack_definition
  variables:
    OS: 'ubuntu'
    DIST: 'eoan'

ubuntu_20_04:
  <<: *pack_definition
  variables:
    OS: 'ubuntu'
    DIST: 'focal'

debian_8:
  <<: *pack_definition
  variables:
    OS: 'debian'
    DIST: 'jessie'

debian_9:
  <<: *pack_definition
  variables:
    OS: 'debian'
    DIST: 'stretch'

debian_10:
  <<: *pack_definition
  variables:
    OS: 'debian'
    DIST: 'buster'

opensuse_15_1:
  <<: *pack_test_definition
  variables:
    OS: 'opensuse-leap'
    DIST: '15.1'

opensuse_15_2:
  <<: *pack_test_definition
  variables:
    OS: 'opensuse-leap'
    DIST: '15.2'

# Deploy

sources_deploy:
  <<: *deploy_definition
  script:
    - ${GITLAB_MAKE} source_deploy

centos_6_deploy:
  <<: *deploy_definition
  variables:
    OS: 'el'
    DIST: '6'

centos_7_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'el'
    DIST: '7'

centos_8_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'el'
    DIST: '8'

fedora_28_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'fedora'
    DIST: '28'

fedora_29_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'fedora'
    DIST: '29'

fedora_30_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'fedora'
    DIST: '30'

fedora_31_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'fedora'
    DIST: '31'

ubuntu_14_04_deploy:
  <<: *deploy_definition
  variables:
    OS: 'ubuntu'
    DIST: 'trusty'

ubuntu_16_04_deploy:
  <<: *deploy_definition
  variables:
    OS: 'ubuntu'
    DIST: 'xenial'

ubuntu_18_04_deploy:
  <<: *deploy_definition
  variables:
    OS: 'ubuntu'
    DIST: 'bionic'

ubuntu_19_10_deploy:
  <<: *deploy_definition
  variables:
    OS: 'ubuntu'
    DIST: 'eoan'

ubuntu_20_04_deploy:
  <<: *deploy_definition
  variables:
    OS: 'ubuntu'
    DIST: 'focal'

debian_8_deploy:
  <<: *deploy_definition
  variables:
    OS: 'debian'
    DIST: 'jessie'

debian_9_deploy:
  <<: *deploy_definition
  variables:
    OS: 'debian'
    DIST: 'stretch'

debian_10_deploy:
  <<: *deploy_definition
  variables:
    OS: 'debian'
    DIST: 'buster'

opensuse_15_1_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'opensuse-leap'
    DIST: '15.1'

opensuse_15_2_deploy:
  <<: *deploy_test_definition
  variables:
    OS: 'opensuse-leap'
    DIST: '15.2'

# Static builds

static_build:
  <<: *docker_test_definition
  script:
    - ${GITLAB_MAKE} test_static_build

static_build_cmake_linux:
  <<: *docker_test_definition
  script:
    - ${GITLAB_MAKE} test_static_build_cmake_linux

static_build_cmake_osx_15:
  <<: *osx_definition
  tags:
    - osx_15
  script:
    - ${GITLAB_MAKE} test_static_build_cmake_osx