user/vijays/vdpa dev

drivers/common/sfc_efx/base/efx.h

@@ -339,6 +339,7 @@ typedef struct efx_mcdi_transport_s {
 	void		(*emt_execute)(void *, efx_mcdi_req_t *);
 	void		(*emt_ev_cpl)(void *);
 	void		(*emt_exception)(void *, efx_mcdi_exception_t);
+	void		(*emt_remap)(void *);
 #if EFSYS_OPT_MCDI_LOGGING
 	void		(*emt_logger)(void *, efx_log_msg_t,
 					void *, size_t, void *, size_t);
@@ -357,6 +358,11 @@ efx_mcdi_init(
 	__in		efx_nic_t *enp,
 	__in		const efx_mcdi_transport_t *mtp);
 
+LIBEFX_API
+extern	__checkReturn	efx_rc_t
+efx_mcdi_dma_remap(
+	__in		efx_nic_t *enp);
+
 LIBEFX_API
 extern	__checkReturn	efx_rc_t
 efx_mcdi_reboot(

drivers/common/sfc_efx/base/efx_mcdi.c

@@ -158,7 +158,49 @@ efx_mcdi_init(
 	return (rc);
 }
 
-			void
+	__checkReturn	efx_rc_t
+efx_mcdi_dma_remap(
+	__in		efx_nic_t *enp)
+{
+	const efx_mcdi_transport_t *emtp = enp->en_mcdi.em_emtp;
+	efx_mcdi_iface_t *emip = &(enp->en_mcdi.em_emip);
+	efx_mcdi_remap_state_t remap_state;
+	efsys_lock_state_t state;
+	efx_rc_t rc;
+
+	if (emtp->emt_remap == NULL) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	EFSYS_LOCK(enp->en_eslp, state);
+	if (emip->emi_pending_req != NULL) {
+		/*
+		 * In-flight MCDI command detected so delay remapping,
+		 * check emip->emi_remap_state in efx_mcdi_request_start()
+		 * and ensure remapping is done before sending next command
+		 */
+		emip->emi_remap_state = EFX_MCDI_DMA_REMAP_STATE_PENDING;
+	}
+	remap_state = emip->emi_remap_state;
+	EFSYS_UNLOCK(enp->en_eslp, state);
+
+	/*
+	 * Don't hold the lock while applying the re-mapping because
+	 * emt_remap invokes the client driver, which can in turn invoke
+	 * the common code again and casue a deadlock
+	 */
+	if (remap_state == EFX_MCDI_DMA_REMAP_STATE_DEFAULT)
+		emtp->emt_remap(emtp->emt_context);
+
+	return 0;
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+	return (rc);
+}
+
+	void
 efx_mcdi_fini(
 	__in		efx_nic_t *enp)
 {
@@ -255,6 +297,7 @@ efx_mcdi_request_start(
 	unsigned int xflags;
 	boolean_t new_epoch;
 	efsys_lock_state_t state;
+	efx_mcdi_remap_state_t remap_state;
 
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_MCDI);
@@ -280,8 +323,19 @@ efx_mcdi_request_start(
 	seq = emip->emi_seq++ & EFX_MASK32(MCDI_HEADER_SEQ);
 	new_epoch = emip->emi_new_epoch;
 	max_version = emip->emi_max_version;
+	if (emip->emi_remap_state == EFX_MCDI_DMA_REMAP_STATE_PENDING)
+		emip->emi_remap_state = EFX_MCDI_DMA_REMAP_STATE_INPROGRESS;
+	remap_state = emip->emi_remap_state;
 	EFSYS_UNLOCK(enp->en_eslp, state);
 
+	if (remap_state == EFX_MCDI_DMA_REMAP_STATE_INPROGRESS) {
+		emtp->emt_remap(emtp->emt_context);
+
+		EFSYS_LOCK(enp->en_eslp, state);
+		emip->emi_remap_state = EFX_MCDI_DMA_REMAP_STATE_DEFAULT;
+		EFSYS_UNLOCK(enp->en_eslp, state);
+	}
+
 	xflags = 0;
 	if (ev_cpl)
 		xflags |= MCDI_HEADER_XFLAGS_EVREQ;

drivers/common/sfc_efx/base/efx_mcdi.h

@@ -25,6 +25,12 @@ extern "C" {
  */
 #define	EFX_MCDI_STATUS_SLEEP_US	10000
 
+typedef enum efx_mcdi_remap_state_e {
+	EFX_MCDI_DMA_REMAP_STATE_DEFAULT,
+	EFX_MCDI_DMA_REMAP_STATE_PENDING,
+	EFX_MCDI_DMA_REMAP_STATE_INPROGRESS,
+} efx_mcdi_remap_state_t;
+
 struct efx_mcdi_req_s {
 	boolean_t	emr_quiet;
 	/* Inputs: Command #, input buffer and length */
@@ -54,6 +60,7 @@ typedef struct efx_mcdi_iface_s {
 	int			emi_aborted;
 	uint32_t		emi_poll_cnt;
 	uint32_t		emi_mc_reboot_status;
+	efx_mcdi_remap_state_t	emi_remap_state;
 } efx_mcdi_iface_t;
 
 LIBEFX_INTERNAL

drivers/common/sfc_efx/sfc_efx_mcdi.c

@@ -264,6 +264,30 @@ sfc_efx_mcdi_ev_proxy_response(void *arg, uint32_t handle, efx_rc_t result)
 	mcdi->proxy_result = result;
 }
 
+static void
+sfc_efx_mcdi_dma_remap(void *arg)
+{
+	struct sfc_efx_mcdi *mcdi = (struct sfc_efx_mcdi *)arg;
+	efx_mcdi_transport_t *emtp;
+	int rc;
+
+	rte_spinlock_lock(&mcdi->lock);
+
+	emtp = &mcdi->transport;
+	rc = mcdi->ops->dma_remap(mcdi->ops_cookie, &mcdi->mem);
+	if (rc != 0)
+		goto fail_dma_alloc;
+
+	emtp->emt_dma_mem = &mcdi->mem;
+
+	rte_spinlock_unlock(&mcdi->lock);
+	return;
+
+fail_dma_alloc:
+	mcdi->state = SFC_EFX_MCDI_DEAD;
+	rte_spinlock_unlock(&mcdi->lock);
+}
+
 int
 sfc_efx_mcdi_init(struct sfc_efx_mcdi *mcdi,
 		  uint32_t logtype, const char *log_prefix, efx_nic_t *nic,
@@ -303,6 +327,7 @@ sfc_efx_mcdi_init(struct sfc_efx_mcdi *mcdi,
 	emtp->emt_exception = sfc_efx_mcdi_exception;
 	emtp->emt_logger = sfc_efx_mcdi_logger;
 	emtp->emt_ev_proxy_response = sfc_efx_mcdi_ev_proxy_response;
+	emtp->emt_remap = sfc_efx_mcdi_dma_remap;
 
 	sfc_efx_mcdi_info(mcdi, "init MCDI");
 	rc = efx_mcdi_init(mcdi->nic, emtp);

drivers/common/sfc_efx/sfc_efx_mcdi.h

@@ -35,6 +35,8 @@ enum sfc_efx_mcdi_state {
 typedef int (sfc_efx_mcdi_dma_alloc_cb)(void *cookie, const char *name,
 					  size_t len, efsys_mem_t *esmp);
 
+typedef int (sfc_efx_mcdi_dma_remap_cb)(void *cookie, efsys_mem_t *esmp);
+
 typedef void (sfc_efx_mcdi_dma_free_cb)(void *cookie, efsys_mem_t *esmp);
 
 typedef void (sfc_efx_mcdi_sched_restart_cb)(void *cookie);
@@ -43,6 +45,7 @@ typedef void (sfc_efx_mcdi_mgmt_evq_poll_cb)(void *cookie);
 
 struct sfc_efx_mcdi_ops {
 	sfc_efx_mcdi_dma_alloc_cb	*dma_alloc;
+	sfc_efx_mcdi_dma_remap_cb	*dma_remap;
 	sfc_efx_mcdi_dma_free_cb	*dma_free;
 	sfc_efx_mcdi_sched_restart_cb	*sched_restart;
 	sfc_efx_mcdi_mgmt_evq_poll_cb	*mgmt_evq_poll;

drivers/common/sfc_efx/version.map

@@ -150,6 +150,7 @@ INTERNAL {
 	efx_mcdi_get_proxy_handle;
 	efx_mcdi_get_timeout;
 	efx_mcdi_init;
+	efx_mcdi_dma_remap;
 	efx_mcdi_mport_alloc_alias;
 	efx_mcdi_new_epoch;
 	efx_mcdi_reboot;

drivers/vdpa/sfc/sfc_vdpa.h

@@ -18,6 +18,14 @@
 
 #define SFC_VDPA_MAC_ADDR			"mac"
 #define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
+#define SFC_SW_VRING_IOVA			0x300000000000
+
+/*
+ * Ensure a gap of at least 0x1000 between the new and existing
+ * IOVA allocations. The gap is not strictly necessary and has
+ * been added just to space out the different IOVA allocations
+ */
+#define SFC_VDPA_IOVA_REMAP_OFFSET		0x1000
 
 /* Broadcast & Unicast MAC filters are supported */
 #define SFC_MAX_SUPPORTED_FILTERS		3
@@ -61,6 +69,7 @@ struct sfc_vdpa_adapter {
 	efsys_bar_t			mem_bar;
 
 	struct sfc_efx_mcdi		mcdi;
+	uint64_t			mcdi_iova;
 	size_t				mcdi_buff_size;
 
 	uint32_t			max_queue_count;
@@ -101,11 +110,15 @@ int
 sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
 		   size_t len, efsys_mem_t *esmp);
 
+int
+sfc_vdpa_dma_remap(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp);
+
 void
 sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp);
 
 int
-sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *vdpa_data, bool do_map);
+sfc_vdpa_dma_map_vhost_mem_table(struct sfc_vdpa_ops_data *vdpa_data,
+				 bool do_map);
 
 int
 sfc_vdpa_filter_remove(struct sfc_vdpa_ops_data *ops_data);

drivers/vdpa/sfc/sfc_vdpa_hw.c

@@ -41,6 +41,7 @@ sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
 
 	sfc_vdpa_log_init(sva, "name=%s, len=%zu", mz_name, len);
 
+	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
 	mz = rte_memzone_reserve_aligned(mz_name, mcdi_buff_size,
 					 numa_node,
 					 RTE_MEMZONE_IOVA_CONTIG,
@@ -52,19 +53,6 @@ sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
 		return -ENOMEM;
 	}
 
-	/* IOVA address for MCDI would be re-calculated if mapping
-	 * using default IOVA would fail.
-	 * TODO: Earlier there was no way to get valid IOVA range.
-	 * Recently a patch has been submitted to get the IOVA range
-	 * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
-	 * in the kernel version >= 5.4. Support to get the default
-	 * IOVA address for MCDI buffer using available IOVA range
-	 * would be added later. Meanwhile default IOVA for MCDI buffer
-	 * is kept at high mem at 2TB. In case of overlap new available
-	 * addresses would be searched and same would be used.
-	 */
-	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
-
 	for (;;) {
 		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
 						 (uint64_t)mz->addr, mcdi_iova,
@@ -84,6 +72,7 @@ sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
 
 	esmp->esm_addr = mcdi_iova;
 	esmp->esm_base = mz->addr;
+	sva->mcdi_iova = mcdi_iova;
 	sva->mcdi_buff_size = mcdi_buff_size;
 
 	sfc_vdpa_info(sva,
@@ -93,6 +82,49 @@ sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
 	return 0;
 }
 
+int
+sfc_vdpa_dma_remap(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp)
+{
+	int ret;
+
+	ret = rte_vfio_container_dma_unmap(sva->vfio_container_fd,
+					   (uint64_t)esmp->esm_base,
+					   esmp->esm_addr,
+					   sva->mcdi_buff_size);
+	if (ret < 0) {
+		sfc_vdpa_err(sva,
+			     "DMA unmap failed for virt=%p iova=0x%lx, len=%zu",
+			     esmp->esm_base, esmp->esm_addr, sva->mcdi_buff_size
+			    );
+		goto error;
+	}
+
+	ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+					 (uint64_t)esmp->esm_base,
+					 sva->mcdi_iova,
+					 sva->mcdi_buff_size);
+
+	if (ret < 0) {
+		sfc_vdpa_err(sva,
+			     "DMA map failed for virt=%p iova=0x%lx, len=%zu",
+			     esmp->esm_base, esmp->esm_addr, sva->mcdi_buff_size
+			    );
+		goto error;
+	}
+
+	esmp->esm_addr = sva->mcdi_iova;
+	sfc_vdpa_info(sva,
+		      "DMA remapping for MCDI done, new IOVA: 0x%lx",
+		      esmp->esm_addr);
+
+	return 0;
+
+error:
+	sfc_vdpa_err(sva, "DMA remapping failed for MCDI : %s",
+		     rte_strerror(rte_errno));
+	return ret;
+}
+
 void
 sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp)
 {
@@ -118,7 +150,8 @@ sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp)
 }
 
 int
-sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *ops_data, bool do_map)
+sfc_vdpa_dma_map_vhost_mem_table(struct sfc_vdpa_ops_data *ops_data,
+				 bool do_map)
 {
 	uint32_t i, j;
 	int rc;

drivers/vdpa/sfc/sfc_vdpa_mcdi.c

@@ -18,6 +18,15 @@ sfc_vdpa_mcdi_dma_alloc(void *cookie, const char *name, size_t len,
 	return sfc_vdpa_dma_alloc(sva, name, len, esmp);
 }
 
+static sfc_efx_mcdi_dma_remap_cb sfc_vdpa_mcdi_dma_remap;
+static int
+sfc_vdpa_mcdi_dma_remap(void *cookie, efsys_mem_t *esmp)
+{
+	struct sfc_vdpa_adapter *sva = cookie;
+
+	return sfc_vdpa_dma_remap(sva, esmp);
+}
+
 static sfc_efx_mcdi_dma_free_cb sfc_vdpa_mcdi_dma_free;
 static void
 sfc_vdpa_mcdi_dma_free(void *cookie, efsys_mem_t *esmp)
@@ -43,6 +52,7 @@ sfc_vdpa_mcdi_mgmt_evq_poll(void *cookie)
 
 static const struct sfc_efx_mcdi_ops sfc_vdpa_mcdi_ops = {
 	.dma_alloc	= sfc_vdpa_mcdi_dma_alloc,
+	.dma_remap	= sfc_vdpa_mcdi_dma_remap,
 	.dma_free	= sfc_vdpa_mcdi_dma_free,
 	.sched_restart  = sfc_vdpa_mcdi_sched_restart,
 	.mgmt_evq_poll  = sfc_vdpa_mcdi_mgmt_evq_poll,