From d1aae5b5500ab8173ed00c67005ad53924bf67ef Mon Sep 17 00:00:00 2001
From: Andrea Lamparelli <a.lamparelli95@gmail.com>
Date: Thu, 25 Jul 2024 08:57:54 +0200
Subject: [PATCH] Update CSI  (#154)

* Update model registry CSI

Signed-off-by: Andrea Lamparelli <a.lamparelli95@gmail.com>

* Add CSI build, tag and push workflow

Signed-off-by: Andrea Lamparelli <a.lamparelli95@gmail.com>

---------

Signed-off-by: Andrea Lamparelli <a.lamparelli95@gmail.com>
---
 .../workflows/csi-build-and-push-image.yml    |  82 +++++
 .github/workflows/csi-test.yml                |  74 +++++
 Dockerfile                                    |   2 +-
 Dockerfile.odh                                |   2 +-
 csi/.gitignore                                |   3 +
 csi/Dockerfile                                |   9 +-
 csi/Dockerfile.dev                            |  45 +++
 csi/GET_STARTED.md                            | 302 +++++++++---------
 csi/Makefile                                  |  69 +++-
 csi/README.md                                 |  14 +-
 csi/go.mod                                    |  46 ++-
 csi/go.sum                                    | 119 +++----
 csi/pkg/storage/modelregistry_provider.go     |   9 +-
 csi/scripts/install_kustomize.sh              |   6 +
 csi/scripts/install_modelregistry.sh          |  14 +-
 csi/test/e2e_test.sh                          | 185 +++++++++++
 csi/test/setup_test_env.sh                    |  48 +++
 17 files changed, 754 insertions(+), 275 deletions(-)
 create mode 100644 .github/workflows/csi-build-and-push-image.yml
 create mode 100644 .github/workflows/csi-test.yml
 create mode 100644 csi/Dockerfile.dev
 create mode 100755 csi/scripts/install_kustomize.sh
 create mode 100755 csi/test/e2e_test.sh
 create mode 100755 csi/test/setup_test_env.sh

diff --git a/.github/workflows/csi-build-and-push-image.yml b/.github/workflows/csi-build-and-push-image.yml
new file mode 100644
index 00000000..02da5ee0
--- /dev/null
+++ b/.github/workflows/csi-build-and-push-image.yml
@@ -0,0 +1,82 @@
+name: CSI container image build and tag
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'v*'
+    paths-ignore:
+      - 'LICENSE*'
+      - '**.gitignore'
+      - '**.md'
+      - '**.txt'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/dependabot.yml'
+      - 'docs/**'
+env:
+  IMG_ORG: kubeflow
+  IMG_REPO: model-registry-storage-initializer
+  DOCKER_USER: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKER_PWD: ${{ secrets.DOCKERHUB_TOKEN }}
+  PUSH_IMAGE: true
+jobs:
+  build-csi-image:
+    runs-on: ubuntu-latest
+    steps:
+      # Assign context variable for various action contexts (tag, main, CI)
+      - name: Assigning tag context
+        if: github.head_ref == '' && startsWith(github.ref, 'refs/tags/v')
+        run: echo "BUILD_CONTEXT=tag" >> $GITHUB_ENV
+      - name: Assigning main context
+        if: github.head_ref == '' && github.ref == 'refs/heads/main'
+        run: echo "BUILD_CONTEXT=main" >> $GITHUB_ENV
+      # checkout branch
+      - uses: actions/checkout@v4
+      # set image version
+      - name: Set main-branch environment
+        if: env.BUILD_CONTEXT == 'main'
+        run: |
+          commit_sha=${{ github.event.after }}
+          tag=main-${commit_sha:0:7}
+          echo "VERSION=${tag}" >> $GITHUB_ENV
+      - name: Set tag environment
+        if: env.BUILD_CONTEXT == 'tag'
+        run: |
+          echo "VERSION=${{ github.ref_name }}" >> $GITHUB_ENV
+      # docker login
+      - name: Docker login
+        shell: bash
+        run: make docker/login
+      # build & push
+      - name: Build CSI Image
+        working-directory: ./csi
+        shell: bash
+        env:
+          IMG_ORG: ${{ env.IMG_ORG }}
+          IMG_REPO: ${{ env.IMG_REPO }}
+          IMG_VERSION: ${{ env.VERSION }}
+        run: |
+          make docker-build-dev
+      - name: Push CSI Image
+        if: env.PUSH_IMAGE == 'true'
+        shell: bash
+        env:
+          IMG: ${{ env.IMG_ORG }}/${{ env.IMG_REPO }}
+        run: IMG=${{ env.IMG }} IMG_VERSION=${{ env.VERSION }} make image/push
+      # Tag latest and main
+      - name: Tag Latest
+        if: env.BUILD_CONTEXT == 'main' && env.PUSH_IMAGE == 'true'
+        shell: bash
+        env:
+          IMG: ${{ env.IMG_ORG }}/${{ env.IMG_REPO }}
+        run: |
+          docker tag ${{ env.IMG }}:$VERSION ${{ env.IMG }}:latest
+          IMG=${{ env.IMG }} IMG_VERSION=latest make image/push
+      - name: Tag Main
+        if: env.BUILD_CONTEXT == 'main' && env.PUSH_IMAGE == 'true'
+        shell: bash
+        env:
+          IMG: ${{ env.IMG_ORG }}/${{ env.IMG_REPO }}
+        run: |
+          docker tag ${{ env.IMG }}:$VERSION ${{ env.IMG }}:main
+          IMG=${{ env.IMG }} IMG_VERSION=main make image/push
diff --git a/.github/workflows/csi-test.yml b/.github/workflows/csi-test.yml
new file mode 100644
index 00000000..74e47dcf
--- /dev/null
+++ b/.github/workflows/csi-test.yml
@@ -0,0 +1,74 @@
+name: Test CSI
+on:
+  push:
+    branches:
+      - "main"
+    paths-ignore:
+      - 'LICENSE*'
+      - '**.gitignore'
+      - '**.md'
+      - '**.txt'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/dependabot.yml'
+      - 'docs/**'
+  pull_request:
+    paths:
+      - "csi/**"
+      - "internal/server/openapi/api_model_registry_service*"
+      - "pkg/openapi/**"
+
+env:
+  IMG_ORG: kubeflow
+  MODEL_REGISTRY_IMG: model-registry
+  MODEL_REGISTRY_CSI_IMG: model-registry-storage-initializer
+  PUSH_IMAGE: false
+  BRANCH: ${{ github.base_ref }}
+jobs:
+  build-and-test-csi-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate tag
+        shell: bash
+        id: tags
+        run: |
+          commit_sha=${{ github.event.after }}
+          tag=main-${commit_sha:0:7}
+          echo "tag=${tag}" >> $GITHUB_OUTPUT
+
+      - name: Install network tools
+        run: sudo apt-get update && sudo apt-get install -y netcat
+
+      - name: Build local model registry image
+        shell: bash
+        env:
+          IMG_REPO: ${{ env.MODEL_REGISTRY_IMG }}
+          VERSION: ${{ steps.tags.outputs.tag }}
+          PUSH_IMAGE: false
+        run: ./scripts/build_deploy.sh
+      
+      - name: Build local custom storage initializer
+        working-directory: ./csi
+        shell: bash
+        env:
+          IMG_REPO: ${{ env.MODEL_REGISTRY_CSI_IMG }}
+          IMG_VERSION: ${{ steps.tags.outputs.tag }}
+        run: make docker-build-dev
+
+      - name: Start KinD cluster
+        uses: helm/kind-action@v1.10.0
+        with:
+          node_image: "kindest/node:v1.27.11"
+
+      - name: Install kustomize
+        run: ./csi/scripts/install_kustomize.sh
+
+      - name: Run tests
+        working-directory: ./csi
+        shell: bash
+        env:
+          MR_IMG: "${{ env.IMG_ORG }}/${{ env.MODEL_REGISTRY_IMG }}:${{ steps.tags.outputs.tag }}"
+          MR_CSI_IMG: "${{ env.IMG_ORG }}/${{ env.MODEL_REGISTRY_CSI_IMG }}:${{ steps.tags.outputs.tag }}"
+          CLUSTER: chart-testing
+        run: ./test/e2e_test.sh
diff --git a/Dockerfile b/Dockerfile
index 4acaa8ff..89a741c5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 # Build the model-registry binary
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
+FROM registry.access.redhat.com/ubi8/go-toolset:1.21 AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
diff --git a/Dockerfile.odh b/Dockerfile.odh
index eed0e0a6..a442a6bb 100644
--- a/Dockerfile.odh
+++ b/Dockerfile.odh
@@ -1,5 +1,5 @@
 # Build the model-registry binary
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
+FROM registry.access.redhat.com/ubi8/go-toolset:1.21 AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
diff --git a/csi/.gitignore b/csi/.gitignore
index 9363314e..06e79e4a 100644
--- a/csi/.gitignore
+++ b/csi/.gitignore
@@ -17,3 +17,6 @@ bin/
 
 # Go workspace file
 go.work
+
+# KServe manifests
+istio-*
\ No newline at end of file
diff --git a/csi/Dockerfile b/csi/Dockerfile
index 7a21bde3..70153975 100644
--- a/csi/Dockerfile
+++ b/csi/Dockerfile
@@ -1,5 +1,8 @@
-# Build the model-registry binary
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
+# This Dockerfile MUST be used if using a fixed version of model-registry dependency
+# This assumes the root is the csi/ folder
+
+# Build the model-registry-storage-initializer binary
+FROM registry.access.redhat.com/ubi8/go-toolset:1.21 AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -14,7 +17,7 @@ USER root
 COPY ["Makefile", "main.go", "./"]
 
 # Copy rest of the source
-COPY bin/ bin/
+# COPY bin/ bin/
 COPY pkg/ pkg/
 
 # Build
diff --git a/csi/Dockerfile.dev b/csi/Dockerfile.dev
new file mode 100644
index 00000000..3f3cf8ec
--- /dev/null
+++ b/csi/Dockerfile.dev
@@ -0,0 +1,45 @@
+# Build the model-registry-storage-initializer binary
+# This assumes the root is the model-registry repository
+FROM registry.access.redhat.com/ubi8/go-toolset:1.21 AS builder
+
+WORKDIR /modelregistry
+# Copy the model-registry Go Modules manifests and sources
+COPY go.mod go.mod
+COPY go.sum go.sum
+COPY cmd/ cmd/
+COPY api/ api/
+COPY internal/ internal/
+COPY scripts/ scripts/
+COPY pkg/ pkg/
+COPY patches/ patches/
+COPY templates/ templates/
+
+USER root
+
+WORKDIR /modelregistry/csi
+# Copy the Go Modules manifests
+COPY csi/go.mod go.mod
+COPY csi/go.sum go.sum
+RUN echo "replace github.com/kubeflow/model-registry => ../" >> go.mod
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy rest of the source
+COPY csi/Makefile Makefile
+COPY csi/main.go main.go
+COPY csi/pkg/ pkg/
+
+# Build
+USER root
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 make build
+
+# Use distroless as minimal base image to package the model-registry storage initializer binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+WORKDIR /
+# copy the storage initializer binary
+COPY --from=builder /modelregistry/csi/bin/mr-storage-initializer .
+USER 65532:65532
+
+ENTRYPOINT ["/mr-storage-initializer"]
\ No newline at end of file
diff --git a/csi/GET_STARTED.md b/csi/GET_STARTED.md
index 2ef7d094..5307c057 100644
--- a/csi/GET_STARTED.md
+++ b/csi/GET_STARTED.md
@@ -15,46 +15,60 @@ We assume all [prerequisites](#prerequisites) are satisfied at this point.
 ### Create the environment
 
 1. After having kind installed, create a kind cluster with:
-```bash
-kind create cluster
-```
+    ```bash
+    kind create cluster
+    ```
 
 2. Configure `kubectl` to use kind context
-```bash
-kubectl config use-context kind-kind
-```
+    ```bash
+    kubectl config use-context kind-kind
+    ```
 
 3. Setup local deployment of *Kserve* using the provided *Kserve quick installation* script
-```bash
-curl -s "https://raw.githubusercontent.com/kserve/kserve/release-0.11/hack/quick_install.sh" | bash
-```
+    ```bash
+    curl -s "https://raw.githubusercontent.com/kserve/kserve/release-0.12/hack/quick_install.sh" | bash
+    ```
 
 4. Install *model registry* in the local cluster
 
-[Optional ]Use model registry with local changes:
-
-```bash
-TAG=$(git rev-parse HEAD) && \
-MR_IMG=quay.io/$USER/model-registry:$TAG && \
-make -C ../ IMG_ORG=$USER IMG_VERSION=$TAG image/build && \
-kind load docker-image $MR_IMG
-```
-
-then:
-
-```bash
-bash ./scripts/install_modelregistry.sh -i $MR_IMG
-```
-
-> _NOTE_: If you want to use a remote image you can simply remove the `-i` option
-
-> _NOTE_: The `./scripts/install_modelregistry.sh` will make some change to [base/kustomization.yaml](../manifests/kustomize/base/kustomization.yaml) that you DON'T need to commit!!
-
-5. [Optional] Use local container image for CSI
-
-```bash
-IMG=quay.io/$USER/model-registry-storage-initializer:$(git rev-parse HEAD) && make IMG=$IMG docker-build && kind load docker-image $IMG
-```
+    [Optional] Use local model registry container image:
+
+    ```bash
+    TAG=$(git rev-parse HEAD) && \
+    MR_IMG=quay.io/$USER/model-registry:$TAG && \
+    make -C ../ IMG_ORG=$USER IMG_VERSION=$TAG image/build && \
+    kind load docker-image $MR_IMG
+    ```
+    or
+    ```bash
+    TAG=... && \
+    MR_IMG=kubeflow/model-registry:$TAG
+    ```
+
+    then:
+
+    ```bash
+    bash ./scripts/install_modelregistry.sh -i $MR_IMG
+    ```
+
+> [!NOTE]
+> The `./scripts/install_modelregistry.sh` will make some change to [base/kustomization.yaml](../manifests/kustomize/base/kustomization.yaml) that you DON'T need to commit!!
+
+5. [Optional] Use local CSI container image
+
+    Either, using the local model-registry library as dependency:
+    ```bash
+    IMG=$USER/model-registry-storage-initializer:$(git rev-parse HEAD) && \ 
+    make IMG=$IMG docker-build-dev && \
+    kind load docker-image $IMG
+    ```
+
+    Or using a fixed version of model-registry library:
+    ```bash
+    IMG=$USER/model-registry-storage-initializer:$(git rev-parse HEAD) && \ 
+    make IMG=$IMG docker-build && \
+    kind load docker-image $IMG
+    ```
 
 ## First InferenceService with ModelRegistry URI
 
@@ -79,68 +93,66 @@ export MR_HOSTNAME=localhost:8080
 
 Then, in the same terminal where you exported `MR_HOSTNAME`, perform the following actions:
 1. Register an empty `RegisteredModel`
-
-```bash
-curl --silent -X 'POST' \
-  "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" \
-  -H 'accept: application/json' \
-  -H 'Content-Type: application/json' \
-  -d '{
-  "description": "Iris scikit-learn model",
-  "name": "iris"
-}'
-```
-
-Expected output:
-```bash
-{"createTimeSinceEpoch":"1709287882361","customProperties":{},"description":"Iris scikit-learn model","id":"1","lastUpdateTimeSinceEpoch":"1709287882361","name":"iris"}
-```
+    ```bash
+    curl --silent -X 'POST' \
+      "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '{
+      "description": "Iris scikit-learn model",
+      "name": "iris"
+    }'
+    ```
+
+    Expected output:
+    ```bash
+    {"createTimeSinceEpoch":"1709287882361","customProperties":{},"description":"Iris scikit-learn model","id":"1","lastUpdateTimeSinceEpoch":"1709287882361","name":"iris"}
+    ```
 
 2. Register the first `ModelVersion`
-
-```bash
-curl --silent -X 'POST' \
-  "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" \
-  -H 'accept: application/json' \
-  -H 'Content-Type: application/json' \
-  -d '{
-  "description": "Iris model version v1",
-  "name": "v1",
-  "registeredModelID": "1"
-}'
-```
-
-Expected output:
-```bash
-{"createTimeSinceEpoch":"1709287890365","customProperties":{},"description":"Iris model version v1","id":"2","lastUpdateTimeSinceEpoch":"1709287890365","name":"v1"}
-```
+    ```bash
+    curl --silent -X 'POST' \
+      "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '{
+      "description": "Iris model version v1",
+      "name": "v1",
+      "registeredModelID": "1"
+    }'
+    ```
+
+    Expected output:
+    ```bash
+    {"createTimeSinceEpoch":"1709287890365","customProperties":{},"description":"Iris model version v1","id":"2","lastUpdateTimeSinceEpoch":"1709287890365","name":"v1"}
+    ```
 
 3. Register the raw `ModelArtifact`
-
-This artifact defines where the actual trained model is stored, i.e., `gs://kfserving-examples/models/sklearn/1.0/model`
-
-```bash
-curl --silent -X 'POST' \
-  "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/2/artifacts" \
-  -H 'accept: application/json' \
-  -H 'Content-Type: application/json' \
-  -d '{
-  "description": "Model artifact for Iris v1",
-  "uri": "gs://kfserving-examples/models/sklearn/1.0/model",
-  "state": "UNKNOWN",
-  "name": "iris-model-v1",
-  "modelFormatName": "sklearn",
-  "modelFormatVersion": "1",
-  "artifactType": "model-artifact"
-}'
-```
-
-Expected output:
-```bash
-{"artifactType":"model-artifact","createTimeSinceEpoch":"1709287972637","customProperties":{},"description":"Model artifact for Iris v1","id":"1","lastUpdateTimeSinceEpoch":"1709287972637","modelFormatName":"sklearn","modelFormatVersion":"1","name":"iris-model-v1","state":"UNKNOWN","uri":"gs://kfserving-examples/models/sklearn/1.0/model"}
-```
-
-> _NOTE_: double check the provided IDs are the expected ones.
+    This artifact defines where the actual trained model is stored, i.e., `gs://kfserving-examples/models/sklearn/1.0/model`
+
+    ```bash
+    curl --silent -X 'POST' \
+      "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/2/artifacts" \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '{
+      "description": "Model artifact for Iris v1",
+      "uri": "gs://kfserving-examples/models/sklearn/1.0/model",
+      "state": "UNKNOWN",
+      "name": "iris-model-v1",
+      "modelFormatName": "sklearn",
+      "modelFormatVersion": "1",
+      "artifactType": "model-artifact"
+    }'
+    ```
+
+    Expected output:
+    ```bash
+    {"artifactType":"model-artifact","createTimeSinceEpoch":"1709287972637","customProperties":{},"description":"Model artifact for Iris v1","id":"1","lastUpdateTimeSinceEpoch":"1709287972637","modelFormatName":"sklearn","modelFormatVersion":"1","name":"iris-model-v1","state":"UNKNOWN","uri":"gs://kfserving-examples/models/sklearn/1.0/model"}
+    ```
+
+> [!NOTE]
+> Double check the provided IDs are the expected ones.
 
 ### Apply the `ClusterStorageContainer` resource
 
@@ -180,70 +192,70 @@ spec:
 EOF
 ```
 
-> _NOTE_: as `$IMG` you could use either the one created during [env preparation](#environment-preparation) or any other remote img in the container registry.
+> [!NOTE]
+> As `$IMG` you could use either the one created during [env preparation](#environment-preparation) or any other remote img in the container registry.
 
 ### Create an `InferenceService`
 
 1. Create a namespace
-```bash
-kubectl create namespace kserve-test
-```
+    ```bash
+    kubectl create namespace kserve-test
+    ```
 
 2. Create the `InferenceService`
-```bash
-kubectl apply -n kserve-test -f - <<EOF
-apiVersion: "serving.kserve.io/v1beta1"
-kind: "InferenceService"
-metadata:
-  name: "iris-model"
-spec:
-  predictor:
-    model:
-      modelFormat:
-        name: sklearn
-      storageUri: "model-registry://iris/v1"
-EOF
-```
+    ```bash
+    kubectl apply -n kserve-test -f - <<EOF
+    apiVersion: "serving.kserve.io/v1beta1"
+    kind: "InferenceService"
+    metadata:
+      name: "iris-model"
+    spec:
+      predictor:
+        model:
+          modelFormat:
+            name: sklearn
+          storageUri: "model-registry://iris/v1"
+    EOF
+    ```
 
 3. Check `InferenceService` status
-```bash
-kubectl get inferenceservices iris-model -n kserve-test
-```
+    ```bash
+    kubectl get inferenceservices iris-model -n kserve-test
+    ```
 
 4. Determine the ingress IP and ports
-
-```bash
-kubectl get svc istio-ingressgateway -n istio-system
-```
-
-And then:
-```bash
-INGRESS_GATEWAY_SERVICE=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
-kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8081:80
-```
-
-After that (in another terminal):
-```bash
-export INGRESS_HOST=localhost
-export INGRESS_PORT=8081
-```
+    ```bash
+    kubectl get svc istio-ingressgateway -n istio-system
+    ```
+
+    And then:
+    ```bash
+    INGRESS_GATEWAY_SERVICE=$(kubectl get svc --namespace istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
+    kubectl port-forward --namespace istio-system svc/${INGRESS_GATEWAY_SERVICE} 8081:80
+    ```
+
+    After that (in another terminal):
+    ```bash
+    export INGRESS_HOST=localhost
+    export INGRESS_PORT=8081
+    ```
 
 5. Perform the inference request
 
-Prepare the input data:
-```bash
-cat <<EOF > "/tmp/iris-input.json"
-{
-  "instances": [
-    [6.8,  2.8,  4.8,  1.4],
-    [6.0,  3.4,  4.5,  1.6]
-  ]
-}
-EOF
-```
-
-If you do not have DNS, you can still curl with the ingress gateway external IP using the HOST Header.
-```bash
-SERVICE_HOSTNAME=$(kubectl get inferenceservice iris-model -n kserve-test -o jsonpath='{.status.url}' | cut -d "/" -f 3)
-curl -v -H "Host: ${SERVICE_HOSTNAME}" -H "Content-Type: application/json" "http://${INGRESS_HOST}:${INGRESS_PORT}/v1/models/iris-v1:predict" -d @/tmp/iris-input.json
-```
+    Prepare the input data:
+    ```bash
+    cat <<EOF > "/tmp/iris-input.json"
+    {
+      "instances": [
+        [6.8,  2.8,  4.8,  1.4],
+        [6.0,  3.4,  4.5,  1.6]
+      ]
+    }
+    EOF
+    ```
+
+    If you do not have DNS, you can still curl with the ingress gateway external IP using the HOST Header.
+    ```bash
+    SERVICE_HOSTNAME=$(kubectl get inferenceservice iris-model -n kserve-test -o jsonpath='{.status.url}' | cut -d "/" -f 3)
+    curl -v -H "Host: ${SERVICE_HOSTNAME}" -H "Content-Type: application/json" "http://${INGRESS_HOST}:${INGRESS_PORT}/v1/models/iris-v1:predict" -d @/tmp/iris-input.json
+    ```
diff --git a/csi/Makefile b/csi/Makefile
index d9f45649..d40da2c4 100644
--- a/csi/Makefile
+++ b/csi/Makefile
@@ -1,4 +1,30 @@
-IMG ?= quay.io/${USER}/model-registry-storage-initializer:latest
+# Useful paths
+MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+PROJECT_PATH := $(patsubst %/,%,$(dir $(MKFILE_PATH)))
+PROJECT_BIN := $(PROJECT_PATH)/bin
+GO := $(PROJECT_BIN)/go1.21.9
+
+# add tools bin directory
+PATH := $(PROJECT_BIN):$(PATH)
+
+# container tool
+DOCKER ?= docker
+DOCKERFILE ?= Dockerfile
+DOCKERFILE_DEV ?= Dockerfile.dev
+
+IMG_REGISTRY ?= 
+# container image organization
+IMG_ORG ?= kubeflow
+# container image version
+IMG_VERSION ?= main
+# container image repository
+IMG_REPO ?= model-registry-storage-initializer
+# container image
+ifdef IMG_REGISTRY
+    IMG := ${IMG_REGISTRY}/${IMG_ORG}/${IMG_REPO}
+else
+    IMG := ${IMG_ORG}/${IMG_REPO}
+endif
 
 .PHONY: help
 help: ## Display this help.
@@ -6,32 +32,47 @@ help: ## Display this help.
 
 ##@ Development
 
+bin-go: ## Install go tool
+	GOBIN=$(PROJECT_BIN) go install golang.org/dl/go1.21.9@latest
+	$(PROJECT_BIN)/go1.21.9 download
+
+.PHONY: bin
+bin: bin-go ## Install required dependencies
+
+.PHONY: tidy
+tidy: ## Run go mod tidy.
+	${GO} mod tidy
+
 .PHONY: fmt
-fmt: ## Run go fmt against code.
-	go fmt ./...
+fmt: bin ## Run go fmt against code.
+	${GO} fmt ./...
 
 .PHONY: vet
-vet: ## Run go vet against code.
-	go vet ./...
+vet: bin ## Run go vet against code.
+	${GO} vet ./...
 
 .PHONY: test
-test: fmt vet ## Run tests.
-	go test ./... -coverprofile cover.out
+test: fmt vet bin ## Run tests.
+	${GO} test ./... -coverprofile cover.out
 
 ##@ Build
 
 .PHONY: build
-build: fmt vet ## Build binary.
-	go build -o bin/mr-storage-initializer main.go
+build: fmt vet bin ## Build binary.
+	${GO} build -o bin/mr-storage-initializer main.go
 
 .PHONY: run
 run: fmt vet ## Run the program
-	go run ./main.go $(SOURCE_URI) $(DEST_PATH)
+	${GO} run ./main.go $(SOURCE_URI) $(DEST_PATH)
 
 .PHONY: docker-build
-docker-build: test ## Build docker image.
-	docker build . -f ./Dockerfile -t ${IMG}
+docker-build: ## Build container image.
+	${DOCKER} build . -f ./$(DOCKERFILE) -t ${IMG}:$(IMG_VERSION)
+
+.PHONY: docker-build-dev
+docker-build-dev: ## Build container image using local model-registry module.
+	cd ../ && ${DOCKER} build . -f ./csi/$(DOCKERFILE_DEV) -t ${IMG}:$(IMG_VERSION)
 
 .PHONY: docker-push
-docker-push: ## Push docker image.
-	docker push ${IMG}
\ No newline at end of file
+docker-push: ## Push container image.
+	${DOCKER} push ${IMG}:$(IMG_VERSION)
\ No newline at end of file
diff --git a/csi/README.md b/csi/README.md
index 329a8189..73f5ae72 100644
--- a/csi/README.md
+++ b/csi/README.md
@@ -56,6 +56,10 @@ You can just run:
 make build
 ```
 
+> [!NOTE]
+> The project is currently using a fixed tag of the root Model Registry. You can use the local one by
+> simply adding `replace github.com/kubeflow/model-registry v0.2.1-alpha => ../` in the `go.mod` file
+
 Which wil create the executable under `bin/mr-storage-initializer`.
 
 ### Run the executable
@@ -70,15 +74,21 @@ or directly running the `main.go` skipping the previous step:
 make SOURCE_URI=model-registry://model/version DEST_PATH=./ run
 ```
 
-> _NOTE_: a Model Registry service should be up and running at `localhost:8080`.
+> [!NOTE]
+> A Model Registry service should be up and running at `localhost:8080`.
 
 ### Build container image
 
-Run:
+Using a fixed version of the model-registry library:
 ```bash
 make docker-build
 ```
 
+Or, using the local model-registry module:
+```bash
+make docker-build-dev
+```
+
 By default the container image name is `quay.io/${USER}/model-registry-storage-initializer:latest` but it can be overridden providing the `IMG` env variable, e.g., `make IMG=abc/ORG/NAME:TAG docker-build`.
 
 ### Push container image
diff --git a/csi/go.mod b/csi/go.mod
index e0b0cda8..f1f6e2fc 100644
--- a/csi/go.mod
+++ b/csi/go.mod
@@ -4,22 +4,21 @@ go 1.21
 
 require (
 	github.com/kserve/kserve v0.12.0
-	github.com/kubeflow/model-registry v0.0.0-20240227075359-247c23c4f6b1
+	github.com/kubeflow/model-registry v0.2.3-alpha
 )
 
 require (
 	cloud.google.com/go v0.112.0 // indirect
-	cloud.google.com/go/compute v1.23.3 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.5 // indirect
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
+	cloud.google.com/go/iam v1.1.6 // indirect
 	cloud.google.com/go/storage v1.36.0 // indirect
 	github.com/aws/aws-sdk-go v1.48.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
@@ -32,27 +31,26 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
-	go.opentelemetry.io/otel v1.21.0 // indirect
-	go.opentelemetry.io/otel/metric v1.21.0 // indirect
-	go.opentelemetry.io/otel/trace v1.21.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/oauth2 v0.20.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/api v0.155.0 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
-	google.golang.org/grpc v1.62.0 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/api v0.162.0 // indirect
+	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/api v0.28.4 // indirect
diff --git a/csi/go.sum b/csi/go.sum
index 920a02d8..d8654577 100644
--- a/csi/go.sum
+++ b/csi/go.sum
@@ -7,13 +7,11 @@ cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK
 cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM=
 cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk=
-cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
-cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
-cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
+cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI=
-cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
+cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc=
+cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI=
 cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8=
 cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -23,8 +21,8 @@ github.com/aws/aws-sdk-go v1.48.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3Tju
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ=
-github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM=
+github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw=
+github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -38,8 +36,9 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZ
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
@@ -63,10 +62,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -75,7 +72,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -120,8 +116,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kserve/kserve v0.12.0 h1:9/H9hGM9UPDDgRXemhlc76nrOEzasNv4Medp1oiKvKE=
 github.com/kserve/kserve v0.12.0/go.mod h1:Rz6mjJFdMIW12dy5enQF767P+xprmltC3+lL2HZwpMY=
-github.com/kubeflow/model-registry v0.0.0-20240227075359-247c23c4f6b1 h1:OA1UDnJPw+CZ839beHuQSQ0MJnnFa99DCixXD79L2m8=
-github.com/kubeflow/model-registry v0.0.0-20240227075359-247c23c4f6b1/go.mod h1:x7xPCKl5wkKKuHV/e4eRKM5VIz/vAEs26+8KlOtrlRs=
+github.com/kubeflow/model-registry v0.2.3-alpha h1:z8eVJu2qw0fWTKaTqUy3faQcLAjRh3rJdaDjGT5Xg4E=
+github.com/kubeflow/model-registry v0.2.3-alpha/go.mod h1:SmCuHJud5UJrau4sNb3T6OBxurRqtzYLDc2Gea2pF5A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -146,27 +142,26 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
-go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
-go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
-go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
-go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
 go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
 go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
-go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
-go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
 go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
@@ -177,9 +172,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
@@ -191,7 +185,6 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -207,15 +200,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
-golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
+golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
+golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -223,9 +214,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -235,22 +225,14 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
@@ -269,7 +251,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
 golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -282,14 +263,12 @@ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEt
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA=
-google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk=
+google.golang.org/api v0.162.0 h1:Vhs54HkaEpkMBdgGdOT2P6F0csGG/vxDS0hWHJzmmps=
+google.golang.org/api v0.162.0/go.mod h1:6SulDkfoBIg4NFmCuZ39XeeAgSHCPecfSUuDyYlAHs0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
-google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -298,12 +277,12 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ=
-google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro=
-google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU=
-google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s=
+google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY=
+google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -311,8 +290,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk=
-google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
+google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
+google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -322,10 +301,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
diff --git a/csi/pkg/storage/modelregistry_provider.go b/csi/pkg/storage/modelregistry_provider.go
index 11729366..01049f69 100644
--- a/csi/pkg/storage/modelregistry_provider.go
+++ b/csi/pkg/storage/modelregistry_provider.go
@@ -56,7 +56,7 @@ func (p *ModelRegistryProvider) DownloadModel(modelDir string, modelName string,
 	// Fetch model version by name or latest if not specified
 	var version *openapi.ModelVersion
 	if versionName != nil {
-		version, _, err = p.Client.ModelRegistryServiceAPI.FindModelVersion(context.Background()).Name(*versionName).ParentResourceID(*model.Id).Execute()
+		version, _, err = p.Client.ModelRegistryServiceAPI.FindModelVersion(context.Background()).Name(*versionName).ParentResourceId(*model.Id).Execute()
 		if err != nil {
 			return err
 		}
@@ -107,12 +107,7 @@ func (p *ModelRegistryProvider) DownloadModel(modelDir string, modelName string,
 		return err
 	}
 
-	modelName = registeredModelName
-	if version.Name != nil {
-		modelName = fmt.Sprintf("%s-%s", modelName, *version.Name)
-	}
-
-	return provider.DownloadModel(modelDir, modelName, *modelArtifact.Uri)
+	return provider.DownloadModel(modelDir, "", *modelArtifact.Uri)
 }
 
 func extractProtocol(storageURI string) (kserve.Protocol, error) {
diff --git a/csi/scripts/install_kustomize.sh b/csi/scripts/install_kustomize.sh
new file mode 100755
index 00000000..04c7ad04
--- /dev/null
+++ b/csi/scripts/install_kustomize.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+curl --silent --location --remote-name "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv5.2.1/kustomize_v5.2.1_linux_amd64.tar.gz"
+tar -xzvf kustomize_v5.2.1_linux_amd64.tar.gz
+chmod a+x kustomize
+sudo mv kustomize /usr/local/bin/kustomize
\ No newline at end of file
diff --git a/csi/scripts/install_modelregistry.sh b/csi/scripts/install_modelregistry.sh
index 8f63c31f..f2ec7317 100755
--- a/csi/scripts/install_modelregistry.sh
+++ b/csi/scripts/install_modelregistry.sh
@@ -10,15 +10,15 @@ Help() {
   echo
   echo "Syntax: [-n NAMESPACE] [-i IMAGE]"
   echo "options:"
-  echo "n Namespace."
-  echo "i Model registry image."
+  echo "  n Namespace."
+  echo "  i Model registry image."
   echo
 }
 
-MR_ROOT=".."
+MR_ROOT=${MR_ROOT:-".."}
 
 namespace=kubeflow
-image=quay.io/opendatahub/model-registry:latest
+image=kubeflow/model-registry:latest
 while getopts ":hn:i:" option; do
    case $option in
       h) # display Help
@@ -40,9 +40,9 @@ if ! kubectl get namespace "$namespace" &> /dev/null; then
 fi
 # Apply model-registry kustomize manifests
 echo Using model registry image: $image
-cd $MR_ROOT/manifests/kustomize/base && kustomize edit set image quay.io/opendatahub/model-registry:latest=${image} && cd -
-kubectl -n $namespace apply -k "$MR_ROOT/manifests/kustomize/overlays/postgres"
+cd $MR_ROOT/manifests/kustomize/base && kustomize edit set image kubeflow/model-registry:latest=${image} && cd -
+kubectl -n $namespace apply -k "$MR_ROOT/manifests/kustomize/overlays/db"
 
 # Wait for model registry deployment
 modelregistry=$(kubectl get pod -n kubeflow --selector="component=model-registry-server" --output jsonpath='{.items[0].metadata.name}')
-kubectl wait --for=condition=Ready pod/$modelregistry -n $namespace --timeout=5m
\ No newline at end of file
+kubectl wait --for=condition=Ready pod/$modelregistry -n $namespace --timeout=6m
\ No newline at end of file
diff --git a/csi/test/e2e_test.sh b/csi/test/e2e_test.sh
new file mode 100755
index 00000000..bcdc47c5
--- /dev/null
+++ b/csi/test/e2e_test.sh
@@ -0,0 +1,185 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# This test assumes there is a Kubernetes environment up and running.
+# It could be either a remote one or a local one (e.g., using KinD or minikube).
+
+# Function to check if the port is ready
+wait_for_port() {
+  local port=$1
+  while ! nc -z localhost $port; do
+    sleep 0.1
+  done
+}
+
+DIR="$(dirname "$0")"
+
+KUBECTL=${KUBECTL:-"kubectl"}
+
+# You can provide a local version of the model registry storage initializer
+# In that case, assure that is visible to the local k8s env, e.g., using 
+# `kind load docker-image $MRCSI_IMG`
+MRCSI_IMG=${MRCSI_IMG:-"kubeflow/model-registry-storage-initializer:main"}
+
+KSERVE_VERSION=${KSERVE_VERSION:-"0.12"}
+MODELREGISTRY_VERSION=${MODELREGISTRY_VERSION:-"v0.2.2-alpha"}
+MODELREGISTRY_CSI=${MODELREGISTRY_CSI:-"v0.2.2-alpha"}
+
+# You can provide a local model registry container image
+MR_IMG=${MR_IMG:-"kubeflow/model-registry:$MODELREGISTRY_VERSION"}
+# You can provide a local model registry storage initializer container image
+MR_CSI_IMG=${MR_CSI_IMG:-"kubeflow/model-registry-storage-initializer:$MODELREGISTRY_CSI"}
+
+# Check if KUBECTL is a valid command
+if [ ! command -v "$KUBECTL" > /dev/null 2>&1 ]; then
+    echo "KUBECTL command not found at: $KUBECTL"
+    exit 1
+fi
+
+if [ ! "$KUBECTL" cluster-info > /dev/null 2>&1 ]; then
+    echo "Cluster not available!"
+    exit 1
+fi
+
+# Setup the environment
+./${DIR}/setup_test_env.sh
+
+# Apply the port forward to access the model registry
+NAMESPACE=${NAMESPACE:-"kubeflow"}
+MR_HOSTNAME=localhost:8080
+MODEL_REGISTRY_SERVICE=model-registry-service
+
+MODEL_REGISTRY_REST_PORT=$(kubectl get svc/$MODEL_REGISTRY_SERVICE -n $NAMESPACE --output jsonpath='{.spec.ports[0].targetPort}')
+
+kubectl port-forward -n $NAMESPACE svc/$MODEL_REGISTRY_SERVICE "8080:$MODEL_REGISTRY_REST_PORT" &
+pf_pid=$!
+
+wait_for_port 8080
+
+echo "Initializing data into Model Registry ..."
+
+curl --silent -X 'POST' \
+  "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+  "description": "Iris scikit-learn model",
+  "name": "iris"
+}'
+
+curl --silent -X 'POST' \
+  "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+  "description": "Iris model version v1",
+  "name": "v1",
+  "registeredModelID": "1"
+}'
+
+curl --silent -X 'POST' \
+  "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/2/artifacts" \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+  "description": "Model artifact for Iris v1",
+  "uri": "gs://kfserving-examples/models/sklearn/1.0/model",
+  "state": "UNKNOWN",
+  "name": "sklearn-iris-v1",
+  "modelFormatName": "sklearn",
+  "modelFormatVersion": "1",
+  "artifactType": "model-artifact"
+}'
+
+echo "======== Model Registry populated ========"
+
+echo "Applying Model Registry custom storage initializer ..."
+
+kubectl apply -f - <<EOF
+apiVersion: "serving.kserve.io/v1alpha1"
+kind: ClusterStorageContainer
+metadata:
+  name: mr-initializer
+spec:
+  container:
+    name: storage-initializer
+    image: $MR_CSI_IMG
+    env:
+    - name: MODEL_REGISTRY_BASE_URL
+      value: "$MODEL_REGISTRY_SERVICE.kubeflow.svc.cluster.local:$MODEL_REGISTRY_REST_PORT"
+    - name: MODEL_REGISTRY_SCHEME
+      value: "http"
+    resources:
+      requests:
+        memory: 100Mi
+        cpu: 100m
+      limits:
+        memory: 1Gi
+        cpu: "1"
+  supportedUriFormats:
+    - prefix: model-registry://
+EOF
+
+echo "======== Custom storage initializer applied ========"
+
+echo "Starting test ..."
+
+KSERVE_TEST_NAMESPACE=kserve-test
+if ! kubectl get namespace $KSERVE_TEST_NAMESPACE &> /dev/null; then
+   kubectl create namespace $KSERVE_TEST_NAMESPACE
+fi
+
+kubectl apply -n $KSERVE_TEST_NAMESPACE -f - <<EOF
+apiVersion: "serving.kserve.io/v1beta1"
+kind: "InferenceService"
+metadata:
+  name: "sklearn-iris"
+spec:
+  predictor:
+    model:
+      modelFormat:
+        name: sklearn
+      storageUri: "model-registry://iris/v1"
+EOF
+
+# wait for pod predictor to be initialized
+sleep 2
+predictor=$(kubectl get pod -n $KSERVE_TEST_NAMESPACE --selector="component=predictor" --output jsonpath='{.items[0].metadata.name}')
+kubectl wait --for=condition=Ready pod/$predictor -n $KSERVE_TEST_NAMESPACE --timeout=5m
+
+INGRESS_GATEWAY_SERVICE=$(kubectl get svc -n istio-system --selector="app=istio-ingressgateway" --output jsonpath='{.items[0].metadata.name}')
+kubectl port-forward -n istio-system svc/${INGRESS_GATEWAY_SERVICE} 8081:80 &
+pf_pid=$!
+
+wait_for_port 8081
+
+INGRESS_HOST="localhost:8081"
+
+cat <<EOF > "/tmp/iris-input.json"
+{
+  "instances": [
+    [6.8,  2.8,  4.8,  1.4],
+    [6.0,  3.4,  4.5,  1.6]
+  ]
+}
+EOF
+
+# kubectl wait --for=condition=Ready inferenceservice/sklearn-iris -n $KSERVE_TEST_NAMESPACE --timeout=5m
+kubectl wait --for=jsonpath='{.status.url}' inferenceservice/sklearn-iris -n $KSERVE_TEST_NAMESPACE --timeout=5m
+sleep 5
+
+SERVICE_HOSTNAME=$(kubectl get inferenceservice sklearn-iris -n $KSERVE_TEST_NAMESPACE -o jsonpath='{.status.url}' | cut -d "/" -f 3)
+res=$(curl -s -H "Host: ${SERVICE_HOSTNAME}" -H "Content-Type: application/json" "http://${INGRESS_HOST}/v1/models/sklearn-iris:predict" -d @/tmp/iris-input.json)
+echo "Received: $res"
+
+if [ ! "$res" = "{\"predictions\":[1,1]}" ]; then
+    echo "Prediction does not match expectation!"
+    echo "Printing some logs for debugging.."
+    kubectl logs pod/$predictor -n $KSERVE_TEST_NAMESPACE -c storage-initializer
+    kubectl logs pod/$predictor -n $KSERVE_TEST_NAMESPACE -c kserve-container
+    exit 1
+else
+    echo "Test succeeded!"
+fi
\ No newline at end of file
diff --git a/csi/test/setup_test_env.sh b/csi/test/setup_test_env.sh
new file mode 100755
index 00000000..6533a23f
--- /dev/null
+++ b/csi/test/setup_test_env.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# This test assumes there is a Kubernetes environment up and running.
+# It could be either a remote one or a local one (e.g., using KinD or minikube).
+
+DIR="$(dirname "$0")"
+
+KUBECTL=${KUBECTL:-"kubectl"}
+CLUSTER=${CLUSTER:-"kind"}
+
+# You can provide a local version of the model registry storage initializer
+# In that case, assure that is visible to the local k8s env, e.g., using 
+# `kind load docker-image $MRCSI_IMG`
+MRCSI_IMG=${MRCSI_IMG:-"kubeflow/model-registry-storage-initializer:main"}
+
+KSERVE_VERSION=${KSERVE_VERSION:-"0.12"}
+MODELREGISTRY_VERSION=${MODELREGISTRY_VERSION:-"v0.2.3-alpha"}
+MODELREGISTRY_CSI=${MODELREGISTRY_CSI:-"v0.2.3-alpha"}
+
+# You can provide a local model registry container image
+MR_IMG=${MR_IMG:-"kubeflow/model-registry:$MODELREGISTRY_VERSION"}
+# You can provide a local model registry storage initializer container image
+MR_CSI_IMG=${MR_CSI_IMG:-"kubeflow/model-registry-storage-initializer:$MODELREGISTRY_CSI"}
+
+# Check if KUBECTL is a valid command
+if [ ! command -v "$KUBECTL" > /dev/null 2>&1 ]; then
+    echo "KUBECTL command not found at: $KUBECTL"
+    exit 1
+fi
+
+if [ ! "$KUBECTL" cluster-info > /dev/null 2>&1 ]; then
+    echo "Cluster not available!"
+    exit 1
+fi
+
+echo "Installing KServe version ${KSERVE_VERSION} ..."
+curl -s "https://raw.githubusercontent.com/kserve/kserve/release-${KSERVE_VERSION}/hack/quick_install.sh" | bash
+echo "============ KServe installed ============"
+
+kind load docker-image -n $CLUSTER $MR_IMG
+kind load docker-image -n $CLUSTER $MR_CSI_IMG
+
+echo "Installing Model Registry ${MR_IMG} ..."
+./${DIR}/../scripts/install_modelregistry.sh -i $MR_IMG
+echo "======== Model Registry installed ========"