From 8272067a464afbba76f806f288b8a8181c9b91a8 Mon Sep 17 00:00:00 2001
From: Kamal Heib <kheib@redhat.com>
Date: Thu, 12 Oct 2023 14:38:08 -0400
Subject: [PATCH] rdma: Add support for running the rdma tests over hardware

The blktests are using soft-RoCE (rdma_rxe) and soft-iWARP (siw) to run
RDMA related tests, this change add support for run nvme-rdma and SRP
tests with real RDMA hardware, this is needed to make sure that we don't
have issues when using real RDMA hardware.

Signed-off-by: Kamal Heib <kheib@redhat.com>
---
 Documentation/running-tests.md |  9 ++++++++-
 common/multipath-over-rdma     | 22 ++++++++++++++++++++--
 tests/nvme/rc                  | 12 ++++++++----
 tests/srp/rc                   |  2 +-
 4 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/Documentation/running-tests.md b/Documentation/running-tests.md
index ae80860c..6bb71cb8 100644
--- a/Documentation/running-tests.md
+++ b/Documentation/running-tests.md
@@ -113,7 +113,7 @@ The NVMe tests can be additionally parameterized via environment variables.
 ### Running nvme-rdma and SRP tests
 
 These tests will use the siw (soft-iWARP) driver by default. The rdma_rxe
-(soft-RoCE) driver is also supported.
+(soft-RoCE) driver and hardware RDMA drivers are also supported.
 
 ```sh
 To use the siw driver:
@@ -123,6 +123,13 @@ nvme_trtype=rdma ./check nvme/
 To use the rdma_rxe driver:
 use_rxe=1 nvme_trtype=rdma ./check nvme/
 use_rxe=1 ./check srp/
+
+To use hardware RDMA drivers, set up hardware RDMA beforehand:
+use_hw_rdma=1 nvme_trtype=rdma ./check nvme/
+use_hw_rdma=1 ./check srp/
+
+Note: The variables use_rxe and use_hw_rdma must not be enabled at the same time.
+Also, Need to make sure to unload the hardware RDMA drivers when testing siw or rdma_rxe.
 ```
 
 ### Normal user
diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma
index ee051002..c72771eb 100644
--- a/common/multipath-over-rdma
+++ b/common/multipath-over-rdma
@@ -13,6 +13,7 @@ fio_aux_path=/tmp/fio-state-files
 memtotal=$(sed -n 's/^MemTotal:[[:blank:]]*\([0-9]*\)[[:blank:]]*kB$/\1/p' /proc/meminfo)
 max_ramdisk_size=$((1<<25))
 use_rxe=${use_rxe:-""}
+use_hw_rdma=${use_hw_rdma:-""}
 ramdisk_size=$((memtotal*(1024/16)))  # in bytes
 if [ $ramdisk_size -gt $max_ramdisk_size ]; then
 	ramdisk_size=$max_ramdisk_size
@@ -439,6 +440,23 @@ stop_soft_rdma() {
 	} >>"$FULL"
 }
 
+start_rdma() {
+	if [[ -n "$use_rxe" && -n "$use_hw_rdma" ]]; then
+		echo "The variables use_rxe and use_hw_rdma must not be enable at the same time."
+		return 1
+	fi
+
+	if [ -z "$use_hw_rdma" ]; then
+		start_soft_rdma
+	fi
+}
+
+stop_rdma() {
+	if [ -z "$use_hw_rdma" ]; then
+		stop_soft_rdma
+	fi
+}
+
 # Look up the block device below the filesystem for directory $1.
 block_dev_of_dir() {
 	df "$1" | {
@@ -580,7 +598,7 @@ configure_null_blk() {
 }
 
 setup_rdma() {
-	start_soft_rdma
+	start_rdma
 	(
 		echo "RDMA interfaces:"
 		cd /sys/class/infiniband &&
@@ -599,7 +617,7 @@ teardown_uncond() {
 	killall -9 multipathd >&/dev/null
 	rm -f /etc/multipath.conf
 	stop_target
-	stop_soft_rdma
+	stop_rdma
 	_exit_null_blk
 }
 
diff --git a/tests/nvme/rc b/tests/nvme/rc
index bac2db72..e22a73c7 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -46,7 +46,7 @@ _nvme_requires() {
 		_have_program rdma
 		if [ -n "$use_rxe" ]; then
 			_have_driver rdma_rxe
-		else
+		elif [ -z "$use_hw_rdma" ]; then
 			_have_driver siw
 		fi
 		;;
@@ -372,7 +372,7 @@ _cleanup_nvmet() {
 	fi
 	modprobe -rq nvmet 2>/dev/null
 	if [[ "${nvme_trtype}" == "rdma" ]]; then
-		stop_soft_rdma
+		stop_rdma
 	fi
 
 	_cleanup_blkdev
@@ -386,7 +386,7 @@ _setup_nvmet() {
 	fi
 	modprobe -q nvme-"${nvme_trtype}"
 	if [[ "${nvme_trtype}" == "rdma" ]]; then
-		start_soft_rdma
+		start_rdma
 		for i in $(rdma_network_interfaces)
 		do
 			if [[ "${nvme_adrfam}" == "ipv6" ]]; then
@@ -554,7 +554,11 @@ _nvme_connect_subsys() {
 		ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
 	fi
 
-	nvme connect "${ARGS[@]}" 2> /dev/null
+	if ! nvme connect "${ARGS[@]}" 2> /dev/null; then
+		if [[ -n "$use_hw_rdma" ]]; then
+			echo "Check RDMA hardware set up; use_hw_rdma is enabledand 'nvme connect' failed."
+		fi
+	fi
 
 	# Wait until device file and uuid/wwid sysfs attributes get ready for
 	# all namespaces.
diff --git a/tests/srp/rc b/tests/srp/rc
index c77ef6c7..a2a8c46a 100755
--- a/tests/srp/rc
+++ b/tests/srp/rc
@@ -53,7 +53,7 @@ group_requires() {
 	_have_module rdma_cm
 	if [ -n "$use_rxe" ]; then
 		_have_module rdma_rxe
-	else
+	elif [ -z "$use_hw_rdma" ]; then
 		_have_module siw
 		_have_kver 5 5
 		_have_iproute2 190404