Merge branch 'main' into add-vm-metrics-train

.github/workflows/collect-data-self-hosted.yml

@@ -83,7 +83,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: use Kepler action to deploy cluster
-        uses: sustainable-computing-io/[email protected].7
+        uses: sustainable-computing-io/[email protected].9
         with:
           ebpfprovider: libbpf
           cluster_provider: kind

.github/workflows/integration-test.yml

@@ -35,15 +35,13 @@ env:
 
 jobs:
   run-integration:
-    if: ${{ (inputs.base_change != 'true') || ( (inputs.base_change == 'true') && (inputs.docker_secret == 'true') ) }}
     runs-on: ubuntu-20.04
     steps:
       - name: use Kepler action to deploy cluster
-        uses: sustainable-computing-io/[email protected].7
+        uses: sustainable-computing-io/[email protected].9
         with:
           runningBranch: kind
           cluster_provider: kind
-          local_dev_cluster_version: v0.0.5
       - name: load kepler image
         run: |
           docker pull ${{ env.KEPLER_IMAGE }}
@@ -74,24 +72,31 @@ jobs:
       - name: test deploying with only estimator
         run: |
           make deploy
-          make e2e-test
+          ./tests/e2e_test.sh --estimator ${{ inputs.additional_opts }}
           make cleanup
         env:
-          OPTS: ESTIMATOR${{ inputs.additional_opts }}
+          OPTS: ESTIMATOR
           KEPLER_IMAGE_VERSION: ${{ inputs.kepler_tag }}
       - name: test deploying with only server
         run: |
           make deploy
-          make e2e-test
+          ./tests/e2e_test.sh --server ${{ inputs.additional_opts }}
           make cleanup
         env:
-          OPTS: SERVER${{ inputs.additional_opts }}
+          OPTS: SERVER
           KEPLER_IMAGE_VERSION: ${{ inputs.kepler_tag }}
       - name: test deploying with estimator and model server
         run: |
           make deploy
-          make e2e-test
+          ./tests/e2e_test.sh --estimator --server ${{ inputs.additional_opts }}
           make cleanup
         env:
-          OPTS: ESTIMATOR SERVER${{ inputs.additional_opts }}
+          OPTS: ESTIMATOR SERVER
           KEPLER_IMAGE_VERSION: ${{ inputs.kepler_tag }}
+
+      - name: upload artifacts on failure
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: integration-test-artifacts
+          path: tmp/e2e

.github/workflows/pr.yml

@@ -208,7 +208,7 @@ jobs:
       image_repo: ${{ vars.IMAGE_REPO || 'docker.io/library' }}
       image_tag: ${{ needs.check-branch.outputs.tag }}
       kepler_tag: release-0.7.11
-      additional_opts: " TEST"
+      additional_opts: --test
 
   integration-test-with-exporter:
     needs: [check-secret, check-branch, check-change, base-image]
@@ -232,7 +232,7 @@ jobs:
       image_repo: ${{ vars.IMAGE_REPO || 'docker.io/library' }}
       image_tag: ${{ needs.check-branch.outputs.tag }}
       kepler_tag: release-0.7.11
-      additional_opts: " DB"
+      additional_opts: --db
 
   integration-test-with-latest:
     needs: [check-secret, check-branch, check-change, base-image]
@@ -244,7 +244,7 @@ jobs:
       image_repo: ${{ vars.IMAGE_REPO || 'docker.io/library' }}
       image_tag: ${{ needs.check-branch.outputs.tag }}
       kepler_tag: latest
-      additional_opts: " DB"
+      additional_opts: --db
 
   verify_model_training_script:
     runs-on: ubuntu-latest

.github/workflows/tekton-test.yml

@@ -38,11 +38,9 @@ env:
   BASE_IMAGE: ${{ inputs.image_repo }}/kepler_model_server_base:${{ inputs.image_tag }}
   IMAGE: ${{ inputs.image_repo }}/kepler_model_server:${{ inputs.image_tag }}
   KIND_CLUSTER_NAME: kind
-  KUBECONFIG: /tmp/kubeconfig
 
 jobs:
   tekton-test:
-    if: ${{ (inputs.base_change != 'true') || ( (inputs.base_change == 'true') && (inputs.docker_secret == 'true') ) }}
     runs-on: ubuntu-latest
     steps:
       - name: checkout
@@ -59,7 +57,7 @@ jobs:
         run: |
           docker build -t $IMAGE -f dockerfiles/Dockerfile .
       - name: use Kepler action to deploy cluster
-        uses: sustainable-computing-io/[email protected].7
+        uses: sustainable-computing-io/[email protected].9
         with:
           ebpfprovider: libbpf
           cluster_provider: kind
@@ -73,6 +71,7 @@ jobs:
       - name: Install Kepler
         working-directory: model_training
         run: |
+          export KUBECONFIG_ROOT_DIR=/tmp/kubeconfig
           ./script.sh deploy_kepler
           ./script.sh deploy_prom_dependency
           podname=$(kubectl get pods -oname -nkepler)
@@ -84,17 +83,20 @@ jobs:
       - name: Prepare PVC
         working-directory: model_training/tekton
         run: |
+          export KUBECONFIG_ROOT_DIR=/tmp/kubeconfig
           kubectl apply -f pvc/hostpath.yaml
 
       - name: Deploy Tasks and Pipelines
         working-directory: model_training/tekton
         run: |
+          export KUBECONFIG_ROOT_DIR=/tmp/kubeconfig
           kubectl apply -f tasks
           kubectl apply -f tasks/s3
           kubectl apply -f pipelines
 
       - name: Run Tekton Collect
         run: |
+          export KUBECONFIG_ROOT_DIR=/tmp/kubeconfig
           cat <<EOF | kubectl apply -f -
           apiVersion: tekton.dev/v1
           kind: PipelineRun
@@ -131,6 +133,7 @@ jobs:
 
       - name: Run Tekton Retrain
         run: |
+          export KUBECONFIG_ROOT_DIR=/tmp/kubeconfig
           cat <<EOF | kubectl apply -f -
           apiVersion: tekton.dev/v1
           kind: PipelineRun
@@ -166,6 +169,7 @@ jobs:
 
       - name: Run Tekton Complete Retrain
         run: |
+          export KUBECONFIG_ROOT_DIR=/tmp/kubeconfig
           cat <<EOF | kubectl apply -f -
           apiVersion: tekton.dev/v1
           kind: PipelineRun

.github/workflows/train-model.yml

@@ -116,7 +116,7 @@ jobs:
               sudo pip3 install awscli
 
       - name: use Kepler action to deploy cluster
-        uses: sustainable-computing-io/[email protected].7
+        uses: sustainable-computing-io/[email protected].9
         with:
           ebpfprovider: libbpf
           cluster_provider: kind

Makefile

@@ -209,10 +209,6 @@ manifest:
 	@./manifests/set.sh "${OPTS}"
 	@$(MAKE) _print
 
-e2e-test:
-	chmod +x ./tests/e2e_test.sh
-	./tests/e2e_test.sh test "${OPTS}"
-
 patch-power-request-client:
 	kubectl patch ds kepler-exporter -n kepler --patch-file ./manifests/test/power-request-client.yaml
 

hack/utils.bash

@@ -0,0 +1,86 @@
+#
+# Copyright 2024.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+is_fn() {
+  [[ $(type -t "$1") == "function" ]]
+  return $?
+}
+
+header() {
+  local title=" 🔆🔆🔆  $*  🔆🔆🔆 "
+
+  local len=40
+  if [[ ${#title} -gt $len ]]; then
+    len=${#title}
+  fi
+
+  echo -e "\n\n  \033[1m${title}\033[0m"
+  echo -n "━━━━━"
+  printf '━%.0s' $(seq "$len")
+  echo "━━━━━━━"
+
+}
+
+info() {
+  echo -e " 🔔 $*" >&2
+}
+
+err() {
+  echo -e " 😱 $*" >&2
+}
+
+warn() {
+  echo -e "   $*" >&2
+}
+
+ok() {
+  echo -e "   ✅ $*" >&2
+}
+
+skip() {
+  echo -e " 🙈 SKIP: $*" >&2
+}
+
+fail() {
+  echo -e " ❌ FAIL: $*" >&2
+}
+
+info_run() {
+  echo -e "      $*\n" >&2
+}
+
+run() {
+  echo -e " ❯ $*\n" >&2
+  "$@"
+}
+
+die() {
+  echo -e "\n ✋ $* "
+  echo -e "──────────────────── ⛔️⛔️⛔️ ────────────────────────\n"
+  exit 1
+}
+
+line() {
+  local len="$1"
+  local style="${2:-thin}"
+  shift
+
+  local ch='─'
+  [[ "$style" == 'heavy' ]] && ch="━"
+
+  printf "$ch%.0s" $(seq "$len") >&2
+  echo
+}