CVE fixes e2e jan5

.buildkite/test-sample-yamls.yml

@@ -2,7 +2,7 @@
 
 - label: 'Test RayCluster Sample YAMLs (nightly operator)'
   instance_size: large
-  image: golang:1.19
+  image: golang:1.20
   commands:
     - ./.buildkite/setup-env.sh
     # Build nightly KubeRay operator image
@@ -14,15 +14,15 @@
 
 - label: 'Test RayCluster Sample YAMLs (latest release)'
   instance_size: large
-  image: golang:1.19
+  image: golang:1.20
   commands:
     - ./.buildkite/setup-env.sh
     # Use KubeRay operator image from the latest release
     - source .venv/bin/activate && BUILDKITE_ENV=true RAY_IMAGE=rayproject/ray:2.7.0 OPERATOR_IMAGE=kuberay/operator:v1.0.0-rc.1 python3 tests/test_sample_raycluster_yamls.py
 
 - label: 'Test RayJob Sample YAMLs (nightly operator)'
   instance_size: large
-  image: golang:1.19
+  image: golang:1.20
   commands:
     - ./.buildkite/setup-env.sh
     # Build nightly KubeRay operator image
@@ -34,15 +34,15 @@
 
 - label: 'Test RayJob Sample YAMLs (latest release)'
   instance_size: large
-  image: golang:1.19
+  image: golang:1.20
   commands:
     - ./.buildkite/setup-env.sh
     # Use KubeRay operator image from the latest release
     - source .venv/bin/activate && BUILDKITE_ENV=true RAY_IMAGE=rayproject/ray:2.7.0 OPERATOR_IMAGE=kuberay/operator:v1.0.0-rc.1 python3 tests/test_sample_rayjob_yamls.py
 
 - label: 'Test RayService Sample YAMLs (nightly operator)'
   instance_size: large
-  image: golang:1.19
+  image: golang:1.20
   commands:
     - ./.buildkite/setup-env.sh
     # Build nightly KubeRay operator image
@@ -54,7 +54,7 @@
 
 - label: 'Test RayService Sample YAMLs (latest release)'
   instance_size: large
-  image: golang:1.19
+  image: golang:1.20
   commands:
     - ./.buildkite/setup-env.sh
     # Use KubeRay operator image from the latest release

.github/workflows/actions/kind/action.yml

@@ -0,0 +1,58 @@
+name: "Set up KinD"
+description: "Step to start and configure KinD cluster"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Init directories
+      shell: bash
+      run: |
+        TEMP_DIR="$(pwd)/tmp"
+        mkdir -p "${TEMP_DIR}"
+        echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
+
+        mkdir -p "$(pwd)/bin"
+        echo "$(pwd)/bin" >> $GITHUB_PATH
+
+    - name: Container image registry
+      shell: bash
+      run: |
+        podman run -d -p 5000:5000 --name registry registry:2.8.1
+
+        export REGISTRY_ADDRESS=$(hostname -i):5000
+        echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
+        echo "Container image registry started at ${REGISTRY_ADDRESS}"
+
+        KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
+        echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
+        envsubst < .github/workflows/actions/kind/kind.yaml > ${KIND_CONFIG_FILE}
+
+        sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
+        [[registry]]
+        prefix = "$REGISTRY_ADDRESS"
+        insecure = true
+        location = "$REGISTRY_ADDRESS"
+        EOF'
+
+    - name: Setup KinD cluster
+      uses: helm/[email protected]
+      with:
+        cluster_name: cluster
+        version: v0.17.0
+        config: ${{ env.KIND_CONFIG_FILE }}
+
+    - name: Print cluster info
+      shell: bash
+      run: |
+        echo "KinD cluster:"
+        kubectl cluster-info
+        kubectl describe nodes
+
+    - name: Install Ingress controller
+      shell: bash
+      run: |
+        VERSION=controller-v1.6.4
+        echo "Deploying Ingress controller into KinD cluster"
+        curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n        - --status-update-interval=10/g" | kubectl apply -f -
+        kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true"
+        kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all

.github/workflows/actions/kind/kind.yaml

@@ -0,0 +1,15 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+containerdConfigPatches:
+  - |-
+    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${REGISTRY_ADDRESS}"]
+      endpoint = ["http://${REGISTRY_ADDRESS}"]

.github/workflows/consistency-check.yaml

@@ -17,11 +17,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Go 1.19.x
-        uses: actions/setup-go@v2
+      - name: Set up Go
+        uses: actions/setup-go@v3
         with:
           # Use the same go version with build job
-          go-version: '1.19'
+          go-version: '1.20'
 
       - name: Check golang version
         working-directory: ./ray-operator
@@ -44,11 +44,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Go 1.19.x
+      - name: Set up Go
         uses: actions/setup-go@v2
         with:
           # Use the same go version with build job
-          go-version: '1.19'
+          go-version: '1.20'
 
       - name: Update CRD/RBAC YAML files
         working-directory: ./ray-operator

.github/workflows/e2e-tests.yaml

@@ -0,0 +1,94 @@
+name: e2e
+
+on:
+  pull_request:
+    branches:
+      - master
+      - 'release-*'
+    paths-ignore:
+      - 'apiserver/**'
+      - 'cli/**'
+      - 'docs/**'
+      - '**.adoc'
+      - '**.md'
+      - 'LICENSE'
+  push:
+    branches:
+      - master
+      - 'release-*'
+    paths-ignore:
+      - 'apiserver/**'
+      - 'cli/**'
+      - 'docs/**'
+      - '**.adoc'
+      - '**.md'
+      - 'LICENSE'
+
+concurrency:
+  group: ${{ github.head_ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  ray-operator:
+
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: v1.20
+
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup and start KinD cluster
+        uses: ./.github/workflows/actions/kind
+
+      - name: Deploy Kuberay operator
+        id: deploy
+        run: |
+          echo Deploying Kuberay operator
+          cd ray-operator
+
+          IMG="${REGISTRY_ADDRESS}"/kuberay
+          make docker-build -e IMG="${IMG}" -e ENGINE=podman
+          make docker-push -e IMG="${IMG}" -e ENGINE=podman
+
+          make deploy -e IMG="${IMG}"
+          kubectl wait --timeout=90s --for=condition=Available=true deployment -n ray-system kuberay-operator
+
+      - name: Run e2e tests
+        run: |
+          export KUBERAY_TEST_TIMEOUT_SHORT=1m
+          export KUBERAY_TEST_TIMEOUT_MEDIUM=5m
+          export KUBERAY_TEST_TIMEOUT_LONG=10m
+
+          export KUBERAY_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
+          echo "KUBERAY_TEST_OUTPUT_DIR=${KUBERAY_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
+
+          set -euo pipefail
+          cd ray-operator
+          go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${KUBERAY_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+
+      - name: Print KubeRay operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing KubeRay operator logs"
+          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${KUBERAY_TEST_OUTPUT_DIR}/kuberay-operator.log
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: always() && steps.deploy.outcome == 'success'
+        with:
+          name: logs
+          retention-days: 10
+          path: |
+            ${{ env.KUBERAY_TEST_OUTPUT_DIR }}/**/*.log

.github/workflows/image-release.yaml

@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.19.x
-      uses: actions/setup-go@v2
+    - name: Set up Go
+      uses: actions/setup-go@v3
       with:
-        go-version: '1.19'
+        go-version: '1.20'
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
@@ -90,10 +90,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.19.x
-      uses: actions/setup-go@v2
+    - name: Set up Go
+      uses: actions/setup-go@v3
       with:
-        go-version: '1.19'
+        go-version: '1.20'
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2

.github/workflows/test-job.yaml

@@ -12,11 +12,11 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.19.x
-      uses: actions/setup-go@v2
+    - name: Set up Go
+      uses: actions/setup-go@v3
       with:
         # Use the same go version with build job
-        go-version: '1.19'
+        go-version: '1.20'
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
@@ -107,10 +107,10 @@ jobs:
     name: Build Apiserver, CLI Binaries and Docker Images
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.19.x
-        uses: actions/setup-go@v2
+      - name: Set up Go
+        uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: '1.20'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -202,10 +202,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.19.x
-      uses: actions/setup-go@v2
+    - name: Set up Go
+      uses: actions/setup-go@v3
       with:
-        go-version: '1.19'
+        go-version: '1.20'
 
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2

apiserver/DEVELOPMENT.md

@@ -5,9 +5,9 @@ This guide covers the purpose, requirements, and deployment of the Kuberay API S
 ## Requirements
 
 | Software | Version  |                                                                Link |
-| :------- | :------: | ------------------------------------------------------------------: |
+|:---------|:--------:|--------------------------------------------------------------------:|
 | kubectl  | v1.18.3+ | [Download](https://kubernetes.io/docs/tasks/tools/install-kubectl/) |
-| Go       |  v1.19   |                                  [Download](https://golang.org/dl/) |
+| Go       |  v1.20   |                                  [Download](https://golang.org/dl/) |
 | Docker   |  19.03+  |                        [Download](https://docs.docker.com/install/) |
 | GNU Make |  3.81+   |                                                                     |
 | curl     |  7.88+   |                                                                     |

apiserver/Dockerfile

@@ -1,5 +1,5 @@
 # Build the backend service
-FROM registry.access.redhat.com/ubi8/go-toolset:1.19.13 as builder
+FROM registry.access.redhat.com/ubi9/go-toolset:1.20.10 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

apiserver/cmd/main.go

@@ -18,7 +18,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/reflection"
 	"google.golang.org/protobuf/encoding/protojson"
-	klog "k8s.io/klog/v2"
+	"k8s.io/klog/v2"
 
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"