ci: switch from jenkins to github actions

.github/bors.toml

@@ -1,5 +1,5 @@
-status = [ "continuous-integration/jenkins/branch" ]
-pr_status = [ "commitlint", "DCO" ]
+status = [ "bors-ci" ]
+pr_status = [ "helm-chart-test", "submodule-branch", "commitlint", "DCO" ]
 timeout_sec = 10000
 required_approvals = 2
 delete_merged_branches = true

.github/workflows/helm-chart.yml

@@ -0,0 +1,33 @@
+name: Helm Chart CI
+on:
+  workflow_call:
+  pull_request:
+    types: ['opened', 'edited', 'reopened', 'synchronize']
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  CI: 1
+
+jobs:
+  helm-chart-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" ./scripts/helm/shell.nix
+      - name: HelmChart publish test
+        run: |
+          nix-shell --pure --run "./scripts/helm/test-publish-chart-yaml.sh" ./scripts/helm/shell.nix
+      - name: HelmChart Readme
+        run: nix-shell --run "./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix
+      - name: HelmChart Template
+        run: nix-shell --pure --run "./scripts/helm/test-template.sh" ./scripts/helm/shell.nix
+

.github/workflows/image-pr.yml

@@ -0,0 +1,23 @@
+name: ImageBuild CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  CI: 1
+
+jobs:
+  image-build-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Test building the release images
+        run: ./scripts/release.sh --skip-publish --build-bins
+      - name: Test building the static binaries
+        run: nix-build -A utils.release.x86_64.linux-musl.kubectl-plugin --arg incremental false
+

.github/workflows/image.yml

@@ -0,0 +1,31 @@
+name: Image Push
+on:
+  push:
+    branches:
+      - develop
+      - 'release/**'
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+**'
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  CI: 1
+
+jobs:
+  image-build-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push the release images
+        run: ./scripts/release.sh
+

.github/workflows/k8s-ci.yml

@@ -1,7 +1,6 @@
 name: K8s CI
 on:
-  pull_request:
-    types: ['opened', 'edited', 'reopened', 'synchronize']
+  workflow_call:
 
 jobs:
   k8s-ci:

.github/workflows/lint.yml

@@ -0,0 +1,35 @@
+name: Linter CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  CI: 1
+
+jobs:
+  linter:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+      - name: Lint rust code
+        run: |
+          nix-shell --run "./dependencies/control-plane/scripts/rust/generate-openapi-bindings.sh"
+          nix-shell --run "./scripts/rust/linter.sh"
+      - name: Lint python code
+        run: nix-shell --run "black --diff --check tests/bdd"
+      - name: Lint nix code
+        run: nix-shell --run "nixpkgs-fmt --check ."
+      - name: Check submodules
+        run: nix-shell --run "./scripts/git/check-submodule-branches.sh"

.github/workflows/nightly-ci.yml

@@ -0,0 +1,16 @@
+name: Nightly CI
+on:
+  workflow_dispatch:
+
+jobs:
+  ci:
+    uses: ./.github/workflows/pr-ci.yml
+  nightly-ci:
+    if: ${{ success() }}
+    needs:
+      - ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: CI succeeded
+        run: exit 0
+

.github/workflows/pr-ci.yml

@@ -0,0 +1,32 @@
+name: Bors CI
+on:
+  workflow_call:
+  push:
+    branches:
+      - staging
+      - trying
+
+jobs:
+  lint-ci:
+    uses: ./.github/workflows/lint.yml
+  helm-ci:
+    uses: ./.github/workflows/helm-chart.yml
+  int-ci:
+    uses: ./.github/workflows/unit-int.yml
+  image-ci:
+    uses: ./.github/workflows/image-pr.yml
+  k8s-ci:
+    uses: ./.github/workflows/k8s-ci.yml
+  bors-ci:
+    if: ${{ success() }}
+    needs:
+      - lint-ci
+      - helm-ci
+      - int-ci
+      - image-ci
+      - k8s-ci
+    runs-on: ubuntu-latest
+    steps:
+      - name: CI succeeded
+        run: exit 0
+

.github/workflows/unit-int.yml

@@ -0,0 +1,42 @@
+name: Integration CI
+on:
+  workflow_call:
+
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  CI: 1
+
+jobs:
+  int-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+      - uses: DeterminateSystems/nix-installer-action@v14
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+      - name: Pre-populate nix-shell
+        run: |
+          export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r)
+          echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV
+          nix-shell --run "echo" shell.nix
+      - name: Handle Rust dependencies caching
+        uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ startsWith(github.ref_name, 'release/') || github.ref_name == 'develop' }}
+      - name: Build the binaries
+        run: nix-shell --run "cargo build --bins"
+      - name: Build the tests
+        run: nix-shell --run "./scripts/rust/test.sh --no-run"
+      - name: Run Tests
+        run: |
+          # includes both unit and integration tests
+          nix-shell --run "./scripts/rust/test.sh"
+# debugging
+      # - name: Setup tmate session
+      #   if: ${{ failure() }}
+      #   timeout-minutes: 120
+      #   uses: mxschmitt/action-tmate@v3
+

scripts/helm/install.sh

@@ -137,7 +137,7 @@ if [ -n "$HOSTED" ]; then
   DEP_UPDATE_ARG=
 fi
 
-if [ "$(helm ls -n openebs -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
+if [ "$(helm ls -n "$K8S_NAMESPACE" -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then
   already_exists_log= "Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE"
   if [ -n "$FAIL_IF_INSTALLED" ]; then
     die "ERROR: $already_exists_log" 1
@@ -154,4 +154,4 @@ else
   set +x
 fi
 
-kubectl get pods -n mayastor -o wide
+kubectl get pods -n "$K8S_NAMESPACE" -o wide

scripts/k8s/deployer.sh

@@ -116,7 +116,9 @@ if [ -z "$COMMAND" ]; then
 fi
 
 if [ "$COMMAND" = "stop" ]; then
-  $SUDO nvme disconnect-all
+  if command -v nvme 2>dev/null; then
+    $SUDO nvme disconnect-all
+  fi
   $KIND delete cluster
   exit 0
 fi

scripts/rust/linter.sh

@@ -29,3 +29,4 @@ if [ -z "$OP" ] || [  "$OP" = "clippy" ]; then
 fi
 
 exit ${FMT_ERROR:-0}
+

scripts/rust/test.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$(dirname "$0")"
+
+ARGS=""
+OPTS=""
+DO_ARGS=
+while [ "$#" -gt 0 ]; do
+  case $1 in
+    --)
+      DO_ARGS="y"
+      shift;;
+    *)
+      if [ "$DO_ARGS" == "y" ]; then
+        ARGS="$ARGS $1"
+      else
+        OPTS="$OPTS $1"
+      fi
+      shift;;
+  esac
+done
+
+set -euxo pipefail
+
+# build test dependencies
+cargo build --bins
+
+cargo test ${OPTS} -- ${ARGS} --test-threads=1
+

tests/helm/test.nix

@@ -39,6 +39,7 @@ pkgs.nixosTest {
         docker
         util-linux
         jq
+        yq-go
         sudo
       ];
       virtualisation = {
@@ -52,9 +53,12 @@ pkgs.nixosTest {
 
   testScript = ''
     machine.wait_for_unit("default.target")
+    machine.succeed("echo ${test-src} > /test-src.txt")
 
-    machine.succeed("${test-src}/scripts/k8s/deployer.sh start --label")
-    machine.succeed("${test-src}/scripts/helm/install.sh --wait")
-    machine.succeed("kubectl get pods -A -o wide 1>&2")
+    machine.succeed("${test-src}/scripts/k8s/deployer.sh start --label --delay")
+    try:
+      machine.succeed("${test-src}/scripts/helm/install.sh --wait")
+    finally:
+      machine.succeed("kubectl get pods -A -o wide 1>&2")
   '';
 }