XDP & AF_XDP for IAVF based on libie and page-pool

drivers/net/ethernet/intel/iavf/iavf.h

@@ -529,6 +529,19 @@ static inline bool iavf_adapter_xdp_active(struct iavf_adapter *adapter)
 	return !!READ_ONCE(adapter->xdp_prog);
 }
 
+static inline struct xsk_buff_pool *iavf_xsk_pool(struct iavf_ring *ring)
+{
+	struct iavf_adapter *adapter = ring->vsi->back;
+	struct iavf_vsi *vsi = ring->vsi;
+	u16 qid = ring->queue_index;
+
+	if (!iavf_adapter_xdp_active(adapter) ||
+	    !test_bit(qid, adapter->af_xdp_zc_qps))
+		return NULL;
+
+	return xsk_get_pool_from_qid(vsi->netdev, qid);
+}
+
 int iavf_up(struct iavf_adapter *adapter);
 void iavf_down(struct iavf_adapter *adapter);
 int iavf_process_config(struct iavf_adapter *adapter);

drivers/net/ethernet/intel/iavf/iavf_main.c

@@ -780,14 +780,29 @@ void iavf_configure_rx_ring(struct iavf_adapter *adapter,
 				       rx_ring->queue_index,
 				       rx_ring->q_vector->napi.napi_id);
 
-	err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_POOL,
-					 rx_ring->pool);
-	if (err)
-		netdev_err(adapter->netdev, "Could not register XDP memory model for RX queue %u, error: %d\n",
-			   queue_idx, err);
+	if (rx_ring->flags & IAVF_TXRX_FLAGS_XSK) {
+		err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq,
+						 MEM_TYPE_XSK_BUFF_POOL,
+						 NULL);
+		if (err)
+			netdev_err(adapter->netdev, "xdp_rxq_info_reg_mem_model returned %d\n",
+				   err);
+
+		xsk_pool_set_rxq_info(rx_ring->xsk_pool, &rx_ring->xdp_rxq);
+
+		iavf_check_alloc_rx_buffers_zc(adapter, rx_ring);
+	} else {
+		err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq,
+						 MEM_TYPE_PAGE_POOL,
+						 rx_ring->pool);
+		if (err)
+			netdev_err(adapter->netdev, "Could not register XDP memory model for RX queue %u, error: %d\n",
+				   queue_idx, err);
+
+		iavf_alloc_rx_pages(rx_ring);
+	}
 
 	RCU_INIT_POINTER(rx_ring->xdp_prog, adapter->xdp_prog);
-	iavf_alloc_rx_pages(rx_ring);
 }
 
 /**
@@ -3657,10 +3672,13 @@ static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
  **/
 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
 {
+	struct iavf_ring *rx_ring;
 	int i, err = 0;
 
 	for (i = 0; i < adapter->num_active_queues; i++) {
-		adapter->rx_rings[i].count = adapter->rx_desc_count;
+		rx_ring = &adapter->rx_rings[i];
+		rx_ring->count = adapter->rx_desc_count;
+
 		err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
 		if (!err)
 			continue;

drivers/net/ethernet/intel/iavf/iavf_trace.h

@@ -145,6 +145,14 @@ DEFINE_EVENT(
 
 	TP_ARGS(ring, desc, skb));
 
+DEFINE_EVENT(
+	iavf_rx_template, iavf_clean_rx_irq_zc,
+	TP_PROTO(struct iavf_ring *ring,
+		 union iavf_32byte_rx_desc *desc,
+		 struct sk_buff *skb),
+
+	TP_ARGS(ring, desc, skb));
+
 DEFINE_EVENT(
 	iavf_rx_template, iavf_clean_rx_irq_rx,
 	TP_PROTO(struct iavf_ring *ring,

drivers/net/ethernet/intel/iavf/iavf_txrx.c

@@ -134,6 +134,11 @@ void iavf_free_tx_resources(struct iavf_ring *tx_ring)
 	kfree(tx_ring->tx_bi);
 	tx_ring->tx_bi = NULL;
 
+	if (tx_ring->flags & IAVF_TXRX_FLAGS_XSK) {
+		tx_ring->dev = tx_ring->xsk_pool->dev;
+		tx_ring->flags &= ~IAVF_TXRX_FLAGS_XSK;
+	}
+
 	if (tx_ring->desc) {
 		dma_free_coherent(tx_ring->dev, tx_ring->size,
 				  tx_ring->desc, tx_ring->dma);
@@ -734,6 +739,22 @@ int iavf_setup_tx_descriptors(struct iavf_ring *tx_ring)
 	return -ENOMEM;
 }
 
+static void iavf_clean_rx_pages(struct iavf_ring *rx_ring)
+{
+	for (u32 i = 0; i < rx_ring->count; i++) {
+		struct page *page = rx_ring->rx_pages[i];
+
+		if (!page)
+			continue;
+
+		/* Invalidate cache lines that may have been written to by
+		 * device so that we avoid corrupting memory.
+		 */
+		page_pool_dma_sync_full_for_cpu(rx_ring->pool, page);
+		page_pool_put_full_page(rx_ring->pool, page, false);
+	}
+}
+
 /**
  * iavf_clean_rx_ring - Free Rx buffers
  * @rx_ring: ring to be cleaned
@@ -749,19 +770,10 @@ void iavf_clean_rx_ring(struct iavf_ring *rx_ring)
 		rx_ring->skb = NULL;
 	}
 
-	/* Free all the Rx ring sk_buffs */
-	for (u32 i = 0; i < rx_ring->count; i++) {
-		struct page *page = rx_ring->rx_pages[i];
-
-		if (!page)
-			continue;
-
-		/* Invalidate cache lines that may have been written to by
-		 * device so that we avoid corrupting memory.
-		 */
-		page_pool_dma_sync_full_for_cpu(rx_ring->pool, page);
-		page_pool_put_full_page(rx_ring->pool, page, false);
-	}
+	if (rx_ring->flags & IAVF_TXRX_FLAGS_XSK)
+		iavf_xsk_clean_rx_ring(rx_ring);
+	else
+		iavf_clean_rx_pages(rx_ring);
 
 	rx_ring->next_to_clean = 0;
 	rx_ring->next_to_use = 0;
@@ -775,7 +787,7 @@ void iavf_clean_rx_ring(struct iavf_ring *rx_ring)
  **/
 void iavf_free_rx_resources(struct iavf_ring *rx_ring)
 {
-	struct device *dev = rx_ring->pool->p.dev;
+	struct device *dev;
 
 	iavf_clean_rx_ring(rx_ring);
 	kfree(rx_ring->rx_pages);
@@ -785,7 +797,14 @@ void iavf_free_rx_resources(struct iavf_ring *rx_ring)
 	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 
-	libie_rx_page_pool_destroy(rx_ring->pool, &rx_ring->rq_stats);
+	if (rx_ring->flags & IAVF_TXRX_FLAGS_XSK) {
+		dev = rx_ring->xsk_pool->dev;
+		rx_ring->flags &= ~IAVF_TXRX_FLAGS_XSK;
+	} else {
+		dev = rx_ring->pool->p.dev;
+		libie_rx_page_pool_destroy(rx_ring->pool, &rx_ring->rq_stats);
+	}
+
 	rx_ring->dev = dev;
 
 	if (rx_ring->desc) {
@@ -820,6 +839,8 @@ int iavf_setup_rx_descriptors(struct iavf_ring *rx_ring)
 
 	/* warn if we are about to overwrite the pointer */
 	WARN_ON(rx_ring->rx_pages);
+
+	/* Both iavf_ring::rx_pages and ::xdp_buff are arrays of pointers */
 	rx_ring->rx_pages = kcalloc(rx_ring->count, sizeof(*rx_ring->rx_pages),
 				    GFP_KERNEL);
 	if (!rx_ring->rx_pages)
@@ -837,6 +858,10 @@ int iavf_setup_rx_descriptors(struct iavf_ring *rx_ring)
 		goto err;
 	}
 
+	iavf_xsk_setup_rx_ring(rx_ring);
+	if (rx_ring->flags & IAVF_TXRX_FLAGS_XSK)
+		goto finish;
+
 	pool = libie_rx_page_pool_create(rx_ring->netdev, rx_ring->count,
 					 iavf_is_xdp_enabled(rx_ring));
 	if (IS_ERR(pool)) {
@@ -846,6 +871,7 @@ int iavf_setup_rx_descriptors(struct iavf_ring *rx_ring)
 
 	rx_ring->pool = pool;
 
+finish:
 	rx_ring->next_to_clean = 0;
 	rx_ring->next_to_use = 0;
 
@@ -860,24 +886,6 @@ int iavf_setup_rx_descriptors(struct iavf_ring *rx_ring)
 	return ret;
 }
 
-/**
- * iavf_release_rx_desc - Store the new tail and head values
- * @rx_ring: ring to bump
- * @val: new head index
- **/
-static inline void iavf_release_rx_desc(struct iavf_ring *rx_ring, u32 val)
-{
-	rx_ring->next_to_use = val;
-
-	/* Force memory writes to complete before letting h/w
-	 * know there are new descriptors to fetch.  (Only
-	 * applicable for weak-ordered memory model archs,
-	 * such as IA-64).
-	 */
-	wmb();
-	writel(val, rx_ring->tail);
-}
-
 /**
  * iavf_receive_skb - Send a completed packet up the stack
  * @rx_ring:  rx ring in play
@@ -1372,9 +1380,7 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget)
 		}
 	}
 
-	libie_rq_napi_stats_add(&rx_ring->rq_stats, &stats);
-	rx_ring->q_vector->rx.total_packets += stats.packets;
-	rx_ring->q_vector->rx.total_bytes += stats.bytes;
+	iavf_update_rx_ring_stats(rx_ring, &stats);
 
 	return cleaned_count;
 }
@@ -1534,7 +1540,9 @@ int iavf_napi_poll(struct napi_struct *napi, int budget)
 	rcu_read_lock();
 
 	iavf_for_each_ring(ring, q_vector->rx) {
-		int cleaned = iavf_clean_rx_irq(ring, budget_per_ring);
+		int cleaned = !!(ring->flags & IAVF_TXRX_FLAGS_XSK) ?
+			      iavf_clean_rx_irq_zc(ring, budget_per_ring) :
+			      iavf_clean_rx_irq(ring, budget_per_ring);
 
 		work_done += cleaned;
 		/* if we clean as many as budgeted, we must not be done */

drivers/net/ethernet/intel/iavf/iavf_txrx.h

@@ -210,6 +210,7 @@ struct iavf_tx_buffer {
 		struct sk_buff *skb;		/* used for .ndo_start_xmit() */
 		struct page *page;		/* used for XDP_TX */
 		struct xdp_frame *xdpf;		/* used for .ndo_xdp_xmit() */
+		struct xdp_buff *xdp;		/* used for XDP_TX in ZC mode */
 	};
 	unsigned int bytecount;
 	unsigned short gso_segs;
@@ -243,6 +244,7 @@ struct iavf_ring {
 	struct net_device *netdev;	/* netdev ring maps to */
 	union {
 		struct iavf_tx_buffer *tx_bi;
+		struct xdp_buff **xdp_buff;
 		struct page **rx_pages;
 	};
 	u8 __iomem *tail;
@@ -370,7 +372,7 @@ static inline __le64 iavf_build_ctob(u32 td_cmd, u32 td_offset,
  * Returns number of data descriptors needed for this skb. Returns 0 to indicate
  * there is not enough descriptors available in this ring since we need at least
  * one descriptor.
- **/
+ */
 static inline int iavf_xmit_descriptor_count(struct sk_buff *skb)
 {
 	const skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
@@ -395,7 +397,7 @@ static inline int iavf_xmit_descriptor_count(struct sk_buff *skb)
  * @size:    the size buffer we want to assure is available
  *
  * Returns 0 if stop is not needed
- **/
+ */
 static inline int iavf_maybe_stop_tx(struct iavf_ring *tx_ring, int size)
 {
 	if (likely(IAVF_DESC_UNUSED(tx_ring) >= size))
@@ -411,7 +413,7 @@ static inline int iavf_maybe_stop_tx(struct iavf_ring *tx_ring, int size)
  * Note: Our HW can't scatter-gather more than 8 fragments to build
  * a packet on the wire and so we need to figure out the cases where we
  * need to linearize the skb.
- **/
+ */
 static inline bool iavf_chk_linearize(struct sk_buff *skb, int count)
 {
 	/* Both TSO and single send will work if count is less than 8 */
@@ -427,7 +429,7 @@ static inline bool iavf_chk_linearize(struct sk_buff *skb, int count)
 /**
  * txring_txq - helper to convert from a ring to a queue
  * @ring: Tx ring to find the netdev equivalent of
- **/
+ */
 static inline struct netdev_queue *txring_txq(const struct iavf_ring *ring)
 {
 	return netdev_get_tx_queue(ring->netdev, ring->queue_index);
@@ -454,7 +456,7 @@ static inline void iavf_xdp_ring_update_tail(const struct iavf_ring *xdp_ring)
  * @tc: TODO
  * @total_pkts: Number of packets transmitted since the last update
  * @total_bytes: Number of bytes transmitted since the last update
- **/
+ */
 static inline void
 __iavf_update_tx_ring_stats(struct iavf_ring *tx_ring,
 			    struct iavf_ring_container *tc,
@@ -468,8 +470,47 @@ __iavf_update_tx_ring_stats(struct iavf_ring *tx_ring,
 #define iavf_update_tx_ring_stats(r, s) \
 	__iavf_update_tx_ring_stats(r, &(r)->q_vector->tx, s)
 
+/**
+ * iavf_update_rx_ring_stats - Update RX ring stats
+ * @rx_ring: ring to bump
+ * @rc: TODO
+ * @rx_bytes: number of bytes processed since last update
+ * @rx_packets: number of packets processed since last update
+ */
+static inline void
+__iavf_update_rx_ring_stats(struct iavf_ring *rx_ring,
+			    struct iavf_ring_container *rc,
+			    const struct libie_rq_onstack_stats *stats)
+{
+	libie_rq_napi_stats_add(&rx_ring->rq_stats, stats);
+	rc->total_packets += stats->packets;
+	rc->total_bytes += stats->bytes;
+}
+
+#define iavf_update_rx_ring_stats(r, s) \
+	__iavf_update_rx_ring_stats(r, &(r)->q_vector->rx, s)
+
+/**
+ * iavf_release_rx_desc - Store the new tail and head values
+ * @rx_ring: ring to bump
+ * @val: new head index
+ */
+static inline void iavf_release_rx_desc(struct iavf_ring *rx_ring, u32 val)
+{
+	rx_ring->next_to_use = val;
+
+	/* Force memory writes to complete before letting h/w
+	 * know there are new descriptors to fetch.  (Only
+	 * applicable for weak-ordered memory model archs,
+	 * such as IA-64).
+	 */
+	wmb();
+	writel(val, rx_ring->tail);
+}
+
 #define IAVF_RXQ_XDP_ACT_FINALIZE_TX	BIT(0)
 #define IAVF_RXQ_XDP_ACT_FINALIZE_REDIR	BIT(1)
+#define IAVF_RXQ_XDP_ACT_STOP_NOW	BIT(2)
 
 /**
  * iavf_set_rs_bit - set RS bit on last produced descriptor.
@@ -495,7 +536,7 @@ static inline u16 iavf_set_rs_bit(struct iavf_ring *xdp_ring)
  * @xdp_ring: XDP TX queue assigned to a given RX ring
  * @rxq_xdp_act: Logical OR of flags of XDP actions that require finalization
  * @first_idx: index of the first frame in the transmitted batch on XDP queue
- **/
+ */
 static inline void iavf_finalize_xdp_rx(struct iavf_ring *xdp_ring,
 					u32 rxq_xdp_act, u32 first_idx)
 {

drivers/net/ethernet/intel/iavf/iavf_virtchnl.c

@@ -415,8 +415,8 @@ static void iavf_set_qp_config_info(struct virtchnl_queue_pair_info *vqpi,
 				    bool xdp_pair)
 {
 	struct iavf_ring *rxq = &adapter->rx_rings[queue_index];
-	const struct page_pool_params *pp = &rxq->pool->p;
 	struct iavf_ring *txq;
+	u32 hr, max_len;
 	int xdpq_idx;
 
 	if (xdp_pair) {
@@ -437,12 +437,20 @@ static void iavf_set_qp_config_info(struct virtchnl_queue_pair_info *vqpi,
 		return;
 	}
 
-	max_frame = min_not_zero(max_frame, LIBIE_MAX_RX_FRM_LEN(pp->offset));
+	if (rxq->flags & IAVF_TXRX_FLAGS_XSK) {
+		hr = xsk_pool_get_headroom(rxq->xsk_pool);
+		max_len = xsk_pool_get_rx_frame_size(rxq->xsk_pool);
+	} else {
+		hr = rxq->pool->p.offset;
+		max_len = rxq->pool->p.max_len;
+	}
+
+	max_frame = min_not_zero(max_frame, LIBIE_MAX_RX_FRM_LEN(hr));
 
 	vqpi->rxq.ring_len = rxq->count;
 	vqpi->rxq.dma_ring_addr = rxq->dma;
 	vqpi->rxq.max_pkt_size = max_frame;
-	vqpi->rxq.databuffer_size = pp->max_len;
+	vqpi->rxq.databuffer_size = max_len;
 }
 
 /**