kungze · jeffyjf · Sep 13, 2022 · Sep 13, 2022
diff --git a/.circleci/01create-local-disks.sh b/.circleci/01create-local-disks.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/bash
+
+set -x
+set -e
+
+sudo apt install qemu-utils tgt open-iscsi
+
+sudo mkdir -p /data
+sudo qemu-img create -f raw /data/rook-ceph-disk01.img 200G
+sudo qemu-img create -f raw /data/rook-ceph-disk02.img 200G
+sudo qemu-img create -f raw /data/rook-ceph-disk03.img 200G
+
+sudo tgtadm --lld iscsi --op new --mode target --tid 1 -T iqn.cinder-metal-csi.kungze.net:localnode
+sudo tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 -b /data/rook-ceph-disk01.img
+sudo tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 2 -b /data/rook-ceph-disk02.img
+sudo tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 3 -b /data/rook-ceph-disk03.img
+sudo tgtadm --lld iscsi --mode target --op bind --tid 1 --initiator-address ALL
+
+sudo iscsiadm -m discovery -t st -p 127.0.0.1
+sudo iscsiadm -m node -T iqn.cinder-metal-csi.kungze.net:localnode --login
diff --git a/.circleci/02setup-k8s-cluster.sh b/.circleci/02setup-k8s-cluster.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/bash
+
+set -x
+set -e
+
+sudo systemctl stop ufw
+sudo modprobe br_netfilter
+sudo swapoff -a
+sudo sysctl -w net.bridge.bridge-nf-call-iptables=1
+sudo sysctl -w net.ipv4.ip_forward=1
+
+sudo apt-get install -y conntrack
+curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+sudo install minikube-linux-amd64 /usr/local/bin/minikube
+
+wget https://storage.googleapis.com/golang/getgo/installer_linux
+chmod +x ./installer_linux
+./installer_linux
+source ~/.bash_profile
+
+git clone https://github.com/Mirantis/cri-dockerd.git
+cd cri-dockerd
+mkdir bin
+go build -o bin/cri-dockerd
+sudo install -o root -g root -m 0755 bin/cri-dockerd /usr/local/bin/cri-dockerd
+sudo cp -a packaging/systemd/* /etc/systemd/system
+sudo sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service
+sudo systemctl daemon-reload
+sudo systemctl enable cri-docker.service
+sudo systemctl enable --now cri-docker.socket
+
+VERSION="v1.25.0"
+wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-amd64.tar.gz
+sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin
+
+wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
+sudo mkdir -p /opt/cni/bin
+sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.1.1.tgz
+
+sudo -E minikube start --driver=none
+
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+chmod +x kubectl
+sudo cp kubectl /usr/local/bin/
+sudo chown circleci:circleci /home/circleci/.kube -R
+sudo chown circleci:circleci /home/circleci/.minikube -R
+
+kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
+
+curl -O https://raw.githubusercontent.com/kubernetes/kubernetes/v1.25.0/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml
+kubedns=`kubectl get svc kube-dns -n kube-system -o jsonpath={.spec.clusterIP}`
+domain="cluster.local"
+localdns="169.254.20.10"
+sed -i "s/__PILLAR__LOCAL__DNS__/$localdns/g; s/__PILLAR__DNS__DOMAIN__/$domain/g; s/,__PILLAR__DNS__SERVER__//g; s/__PILLAR__CLUSTER__DNS__/$kubedns/g" nodelocaldns.yaml
+kubectl apply -f nodelocaldns.yaml
+
+
diff --git a/.circleci/03setup-cinder.sh b/.circleci/03setup-cinder.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/bash
+
+set -x
+set -e
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+git clone --single-branch --branch v1.10.1 https://github.com/rook/rook.git
+cd rook/deploy/examples
+kubectl create -f crds.yaml -f common.yaml -f operator.yaml
+kubectl create -f cluster-test.yaml
+
+get_pod_cmd=(kubectl --namespace rook-ceph get pod --no-headers)
+timeout=450
+start_time="${SECONDS}"
+while [[ $((SECONDS - start_time)) -lt $timeout ]]; do
+    pod="$("${get_pod_cmd[@]}" --selector=app=rook-ceph-osd-prepare --output custom-columns=NAME:.metadata.name,PHASE:status.phase | awk 'FNR <= 1')"
+    if echo "$pod" | grep 'Running\|Succeeded\|Failed'; then break; fi
+    echo 'waiting for at least one osd prepare pod to be running or finished'
+    sleep 5
+done
+
+kubectl -n rook-ceph wait pods -l app=rook-ceph-mon --for condition=Ready --timeout=600s
+
+wget https://get.helm.sh/helm-v3.10.0-rc.1-linux-amd64.tar.gz
+tar -zxvf helm-v3.10.0-rc.1-linux-amd64.tar.gz
+sudo mv linux-amd64/helm /usr/local/bin/helm
+
+docker pull registry.aliyuncs.com/kolla-helm/ubuntu-source-kolla-toolbox:yoga
+docker pull registry.aliyuncs.com/kolla-helm/ubuntu-source-keystone:yoga
+docker pull registry.aliyuncs.com/kolla-helm/ubuntu-source-cinder-backup:yoga
+docker pull registry.aliyuncs.com/kolla-helm/ubuntu-source-cinder-volume:yoga
+docker pull registry.aliyuncs.com/kolla-helm/ubuntu-source-glance-api:yoga
+docker pull registry.aliyuncs.com/kolla-helm/ubuntu-source-kolla-toolbox:yoga
+docker pull registry.aliyuncs.com/kolla-helm/ubuntu-source-cinder-api:yoga
+
+kubectl create namespace openstack
+helm repo add kungze  https://charts.kungze.net
+helm install openstack-password kungze/password --namespace openstack
+helm install openstack-dependency kungze/openstack-dep --namespace openstack --set mariadb.primary.persistence.enabled=false --set rabbitmq.persistence.enabled=false --wait --timeout 2400s
+helm install openstack-keystone kungze/keystone --namespace openstack --wait --timeout 2400s
+helm install openstack-glance kungze/glance --set ceph.replicatedSize=1 --namespace openstack --wait --timeout 2400s
+helm install openstack-cinder kungze/cinder --set lvm.enabled=false --set ceph.failureDomain=osd --set ceph.replicatedSize=1 --set ceph.backup.enabled=false --namespace openstack --wait --timeout 2400s
diff --git a/.circleci/04install-cinder-metal-csi.sh b/.circleci/04install-cinder-metal-csi.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/bash
+
+set -x
+set -e
+
+sudo systemctl stop systemd-resolved.service
+sudo systemctl disable systemd-resolved.service
+sudo rm /etc/resolv.conf
+sudo touch /etc/resolv.conf
+sudo chmod a+w /etc/resolv.conf
+sudo echo "nameserver 169.254.20.10" > /etc/resolv.conf
+sudo echo "nameserver 8.8.8.8" >> /etc/resolv.conf
+
+sudo apt install python3-openstackclient -y
+export OS_USERNAME=$(kubectl get secret -n openstack openstack-keystone -o jsonpath="{.data.OS_USERNAME}" | base64 --decode)
+export OS_PROJECT_DOMAIN_NAME=$(kubectl get secret -n openstack openstack-keystone -o jsonpath="{.data.OS_PROJECT_DOMAIN_NAME}" | base64 --decode)
+export OS_USER_DOMAIN_NAME=$(kubectl get secret -n openstack openstack-keystone -o jsonpath="{.data.OS_USER_DOMAIN_NAME}" | base64 --decode)
+export OS_PROJECT_NAME=$(kubectl get secret -n openstack openstack-keystone -o jsonpath="{.data.OS_PROJECT_NAME}" | base64 --decode)
+export OS_REGION_NAME=$(kubectl get secret -n openstack openstack-keystone -o jsonpath="{.data.OS_REGION_NAME}" | base64 --decode)
+export OS_PASSWORD=$(kubectl get secrets -n openstack openstack-password -o jsonpath="{.data.keystone-admin-password}" | base64 --decode)
+export OS_AUTH_URL=$(kubectl get secret -n openstack openstack-keystone -o jsonpath="{.data.OS_CLUSTER_URL}" | base64 --decode)
+export OS_INTERFACE=internal
+openstack project create kubernetes
+openstack user create --project kubernetes --password ChangeMe kubernetes
+openstack role add --project kubernetes --user kubernetes member
+
+cd  $HOME/project
+export CEPH_CINDER_KEYRING=$(kubectl -n rook-ceph get secrets rook-ceph-client-cinder -o jsonpath="{.data.cinder}" | base64 --decode)
+sed -i 's?__CEPH_CINDER_KEYRING__?'$CEPH_CINDER_KEYRING'?g' manifests/rbd-secrets.yaml
+
+kubectl apply -f manifests/cinder-metal-csi-config.yaml
+kubectl apply -f manifests/cinder-metal-csi-controllerplugin-rbac.yaml
+kubectl apply -f manifests/cinder-metal-csi-controllerplugin.yaml
+kubectl apply -f manifests/cinder-metal-csi-nodeplugin-rbac.yaml
+kubectl apply -f manifests/cinder-metal-csi-nodeplugin.yaml
+kubectl apply -f manifests/rbd-secrets.yaml
+kubectl apply -f manifests/storageclass-rbd.yaml
+kubectl -n kube-system wait pods -l app=cinder-metal-csi-controller-plugin --for condition=Ready --timeout=600s
+kubectl -n kube-system wait pods -l app=cinder-metal-csi-node-plugin --for condition=Ready --timeout=600s
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,43 @@
+version: 2.1
+
+jobs:
+  run-e2e-tests:
+    machine:
+        image: ubuntu-2004:current
+    resource_class: large
+    steps:
+      - checkout
+
+      - run:
+          name: Prepare rook osd block  device
+          command: |
+            ./.circleci/01create-local-disks.sh
+
+      - run:
+          name: Setup k8s cluster
+          command: |
+            ./.circleci/02setup-k8s-cluster.sh
+
+      - run:
+          name: Set cinder service
+          command: |
+            ./.circleci/03setup-cinder.sh
+
+      - run:
+          name: Install cinder-metal-csi plugin
+          command: |
+            ./.circleci/04install-cinder-metal-csi.sh
+
+      - run:
+          name: Run E2E tests
+          command: |
+            export CEPH_CINDER_KEYRING=$(kubectl -n rook-ceph get secrets rook-ceph-client-cinder -o jsonpath="{.data.cinder}" | base64 --decode)
+            cd $HOME/project/tests/e2e
+              go test . -v -args --cinder-volume-type=rbd --ceph-client-keyring=$CEPH_CINDER_KEYRING
+
+
+workflows:
+  e2e-tests:
+    jobs:
+      - run-e2e-tests:
+          type: approval