From 344bb92438702bb42e835adff15b3255be36491a Mon Sep 17 00:00:00 2001 From: Rakshitha Kamath Date: Tue, 5 Dec 2023 11:12:14 +0530 Subject: [PATCH] Add the storage scale server script --- build_scripts/common/basic-server-client.sh | 18 +- build_scripts/common/basic-storage-scale.sh | 197 ++++++++++++++++++ build_scripts/storage-scale/client.sh | 38 ++++ .../storage-scale/download_storage_scale.sh | 3 + jobs/macros.yml | 11 + jobs/scripts/common.sh | 5 + jobs/storage-scale.yml | 40 ++++ 7 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 build_scripts/common/basic-storage-scale.sh create mode 100644 build_scripts/storage-scale/client.sh create mode 100644 jobs/storage-scale.yml diff --git a/build_scripts/common/basic-server-client.sh b/build_scripts/common/basic-server-client.sh index c77e9063..d7b2af68 100644 --- a/build_scripts/common/basic-server-client.sh +++ b/build_scripts/common/basic-server-client.sh @@ -4,6 +4,8 @@ function server_run() { if [ "$JOB_NAME" == "iozone-vfs" ] || [ "$JOB_NAME" == "iozone-vfs-minmdcache" ]; then VOLUME_TYPE="VFS" + elif [ "$JOB_NAME" == "storage-scale" ]; then + VOLUME_TYPE="STORAGE_SCALE" else VOLUME_TYPE="GLUSTER" fi @@ -14,7 +16,12 @@ function server_run() scp ${SSH_OPTIONS} ${2} root@${1}:./$(basename ${2}) - ssh -t ${SSH_OPTIONS} root@${1} "GERRIT_HOST='${GERRIT_HOST}' GERRIT_PROJECT='${GERRIT_PROJECT}' GERRIT_REFSPEC='${GERRIT_REFSPEC}' CENTOS_VERSION='${CENTOS_VERSION}' CENTOS_ARCH='${CENTOS_ARCH}' ${VOLUME_TYPE}_VOLUME='${EXPORT}' YUM_REPO='${YUM_REPO}' ${INCLUDE_TEMPLATE_URL} ${INCLUDE_ACL_PARAM} bash ./$(basename ${2})" + if [ "$JOB_NAME" == "storage-scale" ]; then + ssh -t ${SSH_OPTIONS} root@${1} "AWS_ACCESS_KEY='${AWS_ACCESS_KEY}' AWS_SECRET_KEY='${AWS_SECRET_KEY}' GERRIT_HOST='${GERRIT_HOST}' GERRIT_PROJECT='${GERRIT_PROJECT}' GERRIT_REFSPEC='${GERRIT_REFSPEC}' CENTOS_VERSION='${CENTOS_VERSION}' CENTOS_ARCH='${CENTOS_ARCH}' ${VOLUME_TYPE}_VOLUME='${EXPORT}' YUM_REPO='${YUM_REPO}' ${INCLUDE_TEMPLATE_URL} ${INCLUDE_ACL_PARAM} bash ./$(basename ${2})" + else + ssh -t ${SSH_OPTIONS} root@${1} "GERRIT_HOST='${GERRIT_HOST}' GERRIT_PROJECT='${GERRIT_PROJECT}' GERRIT_REFSPEC='${GERRIT_REFSPEC}' CENTOS_VERSION='${CENTOS_VERSION}' CENTOS_ARCH='${CENTOS_ARCH}' ${VOLUME_TYPE}_VOLUME='${EXPORT}' YUM_REPO='${YUM_REPO}' ${INCLUDE_TEMPLATE_URL} ${INCLUDE_ACL_PARAM} bash ./$(basename ${2})" + fi + #RETURN_CODE=$? #return $RETURN_CODE @@ -28,8 +35,13 @@ function client_run() INCLUDE_TEST_PARAMS="TEST_PARAMETERS='${TEST_PARAMETERS}'" fi - ssh -t ${SSH_OPTIONS} root@${1} "SERVER='${3}' EXPORT='/${EXPORT}' CENTOS_VERSION='${CENTOS_VERSION}' ${INCLUDE_TEST_PARAMS} bash ./$(basename ${2})" - RETURN_CODE_CLIENT=$? + if [ "$JOB_NAME" == "storage-scale" ]; then + ssh -t ${SSH_OPTIONS} root@${1} "SERVER='${3}' EXPORT='/ibm/${EXPORT}' CENTOS_VERSION='${CENTOS_VERSION}' ${INCLUDE_TEST_PARAMS} bash ./$(basename ${2})" + RETURN_CODE_CLIENT=$? + else + ssh -t ${SSH_OPTIONS} root@${1} "SERVER='${3}' EXPORT='/${EXPORT}' CENTOS_VERSION='${CENTOS_VERSION}' ${INCLUDE_TEST_PARAMS} bash ./$(basename ${2})" + RETURN_CODE_CLIENT=$? + fi return $RETURN_CODE_CLIENT } diff --git a/build_scripts/common/basic-storage-scale.sh b/build_scripts/common/basic-storage-scale.sh new file mode 100644 index 00000000..793ec6c2 --- /dev/null +++ b/build_scripts/common/basic-storage-scale.sh @@ -0,0 +1,197 @@ +#!/bin/sh +# +# Setup a simple gluster environment and export a volume through NFS-Ganesha. +# +# This script uses the following environment variables:/ +# - GLUSTER_VOLUME: name of the gluster volume to create +# this name will also be used as name for the export +# +# The YUM_REPO and GERRIT_* variables are mutually exclusive. +# +# - YUM_REPO: URL to the yum repository (.repo file) for the NFS-Ganesha +# packages. When this option is used, libntirpc-latest is enabled +# as well. Leave empty in case patches from Gerrit need testing. +# +# - GERRIT_HOST: when triggered from a new patch submission, this is set to the +# git server that contains the repository to use. +# +# - GERRIT_PROJECT: project that triggered the build (like ffilz/nfs-ganesha). +# +# - GERRIT_REFSPEC: git tree-ish that can be fetched and checked-out for testing. + +set -x + +#THE FOLLOWING LINES OF CODE DOWNLOADS STORAGE SCALE, INSTALLS IT AND CREATES A CLUSTER +#---------------------------------------------------------------------------------------------- +WORKING_DIR="DOWNLOAD_STORAGE_SCALE" +mkdir -p $WORKING_DIR +cd $WORKING_DIR +echo $PWD +yum install -y unzip +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +ls -ltr +unzip -qq awscliv2.zip +chmod +x ./aws/* +./aws/install +aws --version +aws configure set aws_access_key_id ${AWS_ACCESS_KEY} +aws configure set aws_secret_access_key ${AWS_SECRET_KEY} +aws s3api get-object --bucket nfsganesha-ci --key "version_to_use.txt" "version_to_use.txt" +VERSION_TO_USE=$(cat version_to_use.txt) +echo ${VERSION_TO_USE} +aws s3api get-object --bucket nfsganesha-ci --key "${VERSION_TO_USE}" "Storage_Scale_Developer-5.1.9.0-x86_64-Linux-install.zip" +unzip Storage_Scale_Developer-5.1.9.0-x86_64-Linux-install.zip + +ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +chmod og-wx ~/.ssh/authorized_keys + +yum -y install kernel-devel cpp gcc gcc-c++ binutils numactl jre make elfutils elfutils-devel rpcbind sssd-tools openldap-clients bind-utils net-tools krb5-workstation python3 +python3 -m pip install --user ansible +yum install -y kernel-devel-4.18.0-499.el8.x86_64 kernel-headers-4.18.0-499.el8.x86_64 + +#Add CES IP to /etc/hosts +ip_address=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) + +for new_ip in $(echo $ip_address | awk -F '.' '{for(i=$4+1;i<=255;i++){print $1"."$2"."$3"."i}}'); do ping -c 2 $new_ip; if [ "$?" == "1" ]; then USABLE_IP=$new_ip; break; fi; done + +echo "$USABLE_IP cesip1" >> /etc/hosts + +INSTALLER_VERSION=$(echo ${VERSION_TO_USE/.zip/}) +chmod +x $INSTALLER_VERSION +./$INSTALLER_VERSION --silent + +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale setup -s 127.0.0.1 --storesecret; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale node add $(hostname) -n; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale node add $(hostname) -p; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale config protocols -e $USABLE_IP; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale node add -a $(hostname); +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale config gpfs -c $(hostname)_cluster; +dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a1 bs=1M count=8192; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale nsd add -p $(hostname) -u dataAndMetadata -fs ${STORAGE_SCALE_VOLUME} -fg 1 /home/nsd1_c84f2u09-rhel88a1; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale config protocols -f ${STORAGE_SCALE_VOLUME} -m /ibm/${STORAGE_SCALE_VOLUME}; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale enable nfs; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale enable smb; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale callhome disable; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale config perfmon -r off; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale node list; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale install --precheck; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale install; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale deploy --precheck; +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale deploy; + +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale nsd list +/usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale filesystem list +#---------------------------------------------------------------------------------------------- + +#THE FOLLOWING LINES OF CODE CLONES THE SOURCE CODE, RPMBUILD AND INSTALLS THE RPMS +#---------------------------------------------------------------------------------------------- +# make sure rpcbind is running +yum -y install rpcbind +systemctl start rpcbind + +echo 'TODO: this is BAD, needs a fix in the selinux-policy' +setenforce 0 + +systemctl stop firewalld || true + +# enable repositories +yum -y install centos-release-gluster yum-utils centos-release-ceph epel-release unzip + +if [ -n "${YUM_REPO}" ] +then + yum-config-manager --add-repo=http://artifacts.ci.centos.org/nfs-ganesha/nightly/libntirpc/libntirpc-latest.repo + yum-config-manager --add-repo=${YUM_REPO} + + # install the latest version of gluster + yum -y install nfs-ganesha nfs-ganesha-gluster glusterfs-ganesha + + # start nfs-ganesha service + if ! systemctl start nfs-ganesha + then + echo "+++ systemctl status nfs-ganesha.service +++" + systemctl status nfs-ganesha.service + echo "+++ journalctl -xe +++" + journalctl -xe + exit 1 + fi +else + [ -n "${GERRIT_HOST}" ] + [ -n "${GERRIT_PROJECT}" ] + [ -n "${GERRIT_REFSPEC}" ] + + GIT_REPO=$(basename "${GERRIT_PROJECT}") + GIT_URL="https://${GERRIT_HOST}/${GERRIT_PROJECT}" + + BASE_PACKAGES="git bison flex cmake gcc-c++ libacl-devel krb5-devel dbus-devel rpm-build redhat-rpm-config gdb" + BUILDREQUIRES_EXTRA="libnsl2-devel libnfsidmap-devel libwbclient-devel userspace-rcu-devel libcephfs-devel" + if [ "${CENTOS_VERSION}" = "7" ]; then + yum -y install libgfapi-devel + yum -y install ${BASE_PACKAGES} libnfsidmap-devel libwbclient-devel libcap-devel libblkid-devel userspace-rcu-devel userspace-rcu python2-devel + elif [ "${CENTOS_VERSION}" = "8s" ]; then + yum install -y ${BASE_PACKAGES} libacl-devel libblkid-devel libcap-devel redhat-rpm-config rpm-build libgfapi-devel xfsprogs-devel + yum install --enablerepo=powertools -y ${BUILDREQUIRES_EXTRA} + yum -y install selinux-policy-devel sqlite + elif [ "${CENTOS_VERSION}" = "9s" ]; then + yum install -y ${BASE_PACKAGES} libacl-devel libblkid-devel libcap-devel redhat-rpm-config rpm-build libgfapi-devel xfsprogs-devel + yum install --enablerepo=crb -y ${BUILDREQUIRES_EXTRA} + yum -y install selinux-policy-devel sqlite + fi + + git init "${GIT_REPO}" + pushd "${GIT_REPO}" + + #Its observed that fetch is failing so this little hack is added! Will delete in future if it turns out useless! + git fetch --depth=1 "${GIT_URL}" "${GERRIT_REFSPEC}" > /dev/null + if [ $? = 0 ]; then + echo "Fetch succeeded" + else + sleep 2 + git fetch "${GIT_URL}" "${GERRIT_REFSPEC}" + fi + + git checkout -b "${GERRIT_REFSPEC}" FETCH_HEAD + + # update libntirpc + git submodule update --init || git submodule sync + + mkdir build + pushd build + + cmake -DCMAKE_BUILD_TYPE=Maintainer -DUSE_FSAL_GPFS=ON -DUSE_DBUS=ON -D_MSPAC_SUPPORT=OFF ../src + make dist + rpmbuild -ta --define "_srcrpmdir $PWD" --define "_rpmdir $PWD" *.tar.gz + rpm_arch=$(rpm -E '%{_arch}') + ganesha_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p *.src.rpm) + if [ -e ${rpm_arch}/libntirpc-devel*.rpm ]; then + ntirpc_version=$(rpm -q --qf '%{VERSION}-%{RELEASE}' -p ${rpm_arch}/libntirpc-devel*.rpm) + ntirpc_rpm=${rpm_arch}/libntirpc-${ntirpc_version}.${rpm_arch}.rpm + fi + yum -y install {x86_64,noarch}/*.rpm + + #Test block + ulimit -a + ulimit -c unlimited + ulimit -a + + # start nfs-ganesha service with an empty configuration + echo "NFSv4 { Graceless = true; }" > /etc/ganesha/ganesha.conf + if ! systemctl start nfs-ganesha + then + echo "+++ systemctl status nfs-ganesha.service +++" + systemctl status nfs-ganesha.service + echo "+++ journalctl -xe +++" + journalctl -xe + exit 1 + fi +fi +#---------------------------------------------------------------------------------------------- + + +#EXPORT THE NFS VOLUME +#---------------------------------------------------------------------------------------------- +/usr/lpp/mmfs/bin/mmuserauth service create --data-access-method file --type userdefined +/usr/lpp/mmfs/bin/mmnfs export add /ibm/${STORAGE_SCALE_VOLUME} -c "*(Access_Type=RW,Squash=none)" + +#CHECKS TO SEE IF THE VOLUME IS WORKING +#---------------------------------------------------------------------------------------------- diff --git a/build_scripts/storage-scale/client.sh b/build_scripts/storage-scale/client.sh new file mode 100644 index 00000000..58017697 --- /dev/null +++ b/build_scripts/storage-scale/client.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# +# Environment variables used: +# - SERVER: hostname or IP-address of the NFS-server +# - EXPORT: NFS-export to test (should start with "/") + +# if any command fails, the script should exit +set -e + +# enable some more output +set -x + +[ -n "${SERVER}" ] +[ -n "${EXPORT}" ] + +# install build and runtime dependencies +yum -y install git gcc nfs-utils time make + +if [ "${CENTOS_VERSION}" == "8s" ]; then + ENABLE_REPO="--enablerepo=powertools" +elif [ "${CENTOS_VERSION}" == "9s" ]; then + ENABLE_REPO="--enablerepo=crb" +fi +yum ${ENABLE_REPO} install -y libtirpc-devel + +#Logic to generate corefiles +echo "/tmp/cores/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern +mkdir -p /tmp/cores + +# checkout the connectathon tests +git clone --depth=1 git://git.linux-nfs.org/projects/steved/cthon04.git +cd cthon04 +make all + +# v3 mount +mkdir -p /mnt/nfsv3 +mount -t nfs -o vers=3 ${SERVER}:${EXPORT} /mnt/nfsv3 +./server -a -p ${EXPORT} -m /mnt/nfsv3 ${SERVER} diff --git a/build_scripts/storage-scale/download_storage_scale.sh b/build_scripts/storage-scale/download_storage_scale.sh index dbab8ed1..a83c092a 100644 --- a/build_scripts/storage-scale/download_storage_scale.sh +++ b/build_scripts/storage-scale/download_storage_scale.sh @@ -62,4 +62,7 @@ dd if=/dev/zero of=/home/nsd1_c84f2u09-rhel88a1 bs=1M count=8192; /usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale nsd list /usr/lpp/mmfs/5.1.9.0/ansible-toolkit/spectrumscale filesystem list +/usr/lpp/mmfs/bin/mmuserauth service create --data-access-method file --type userdefined +/usr/lpp/mmfs/bin/mmnfs export add /ibm/fs1/ -c "*(Access_Type=RW,Squash=none)" + sleep 6000 diff --git a/jobs/macros.yml b/jobs/macros.yml index d8f187e6..f618820b 100644 --- a/jobs/macros.yml +++ b/jobs/macros.yml @@ -129,6 +129,17 @@ credential-id: GERRITHUB_PRIVATE_KEY variable: GERRITHUB_KEY +- wrapper: + name: aws_credentials + wrappers: + - credentials-binding: + - text: + credential-id: AWS_ACCESS_KEY + variable: ACCESS_KEY + - text: + credential-id: AWS_SECRET_KEY + variable: SECRET_KEY + #Enable this to over-ride the configuration under Manage Jenkins > Configure System > Global Build Time Out #This one is meant for pynfs jobs, setting the value to 4 hours as a part of my testing! - wrapper: diff --git a/jobs/scripts/common.sh b/jobs/scripts/common.sh index 93e10b57..b813c9b8 100644 --- a/jobs/scripts/common.sh +++ b/jobs/scripts/common.sh @@ -6,6 +6,11 @@ export GERRIT_REFSPEC=${GERRIT_REFSPEC} export LAST_TRIGGERED_JOB_NAME=$JOB_NAME export BUILD_NUMBER=${BUILD_NUMBER} +if [ "$JOB_NAME" == "storage-scale" ]; then + export AWS_ACCESS_KEY=${ACCESS_KEY} + export AWS_SECRET_KEY=${SECRET_KEY} +fi + bash $WORKSPACE/ci-tests/build_scripts/common/basic-server-client.sh RET=$? diff --git a/jobs/storage-scale.yml b/jobs/storage-scale.yml new file mode 100644 index 00000000..52cdc09d --- /dev/null +++ b/jobs/storage-scale.yml @@ -0,0 +1,40 @@ +- job: + name: storage-scale + display-name: "Storage Scale" + node: cico-workspace + description: 'Run Storage Scale CI test' + project-type: freestyle + concurrent: false + allow-manual-triggers: true + + scm: + - ci-tests + + properties: + - discarder + + parameters: + - string: + default: "$WORKSPACE/ci-tests/build_scripts/common/basic-storage-scale.sh" + description: Test script to execute on the reserved machine acting as a server. + name: SERVER_TEST_SCRIPT + - string: + default: "$WORKSPACE/ci-tests/build_scripts/storage-scale/client.sh" + description: Test script to execute on the reserved machine acting as a client. + name: CLIENT_TEST_SCRIPT + - nfs_variables: + export_var: 'storage_scale' + - gerrit_variables + - centos_variables + + builders: + - get-node + - shell: !include-raw: scripts/common.sh + + wrappers: + - cleanup-ws + - gerrithub_key + - aws_credentials + + publishers: + - post_build_task_return-node