FIXUP (TO SQUASH): Ensure dma*sync* calls are balanced

Move DMA sync calls into map()/unmap() functions and ensure
they are balanced.

Signed-off-by: Nick Hollinghurst <[email protected]>

Author: njhollinghurst

drivers/iommu/bcm2712-iommu.c

@@ -216,13 +216,6 @@ static int bcm2712_iommu_init(struct bcm2712_iommu *mmu)
 		return -ENOMEM;
 	}
 	MMU_WR(MMMU_ILLEGAL_ADR_OFFSET, MMMU_ILLEGAL_ADR_ENABLE + u);
-	dma_sync_single_for_device(mmu->dev,
-				   virt_to_phys(mmu->top_table),
-				   PAGE_SIZE, DMA_TO_DEVICE);
-	dma_sync_single_for_device(mmu->dev,
-				   virt_to_phys(mmu->default_page),
-				   PAGE_SIZE, DMA_TO_DEVICE);
-	mmu->dirty_top = false;
 	mmu->nmapped_pages = 0;
 
 	/* Flush (and enable) the shared TLB cache; enable this MMU. */
@@ -267,16 +260,21 @@ static int bcm2712_iommu_map(struct iommu_domain *domain, unsigned long iova,
 	struct bcm2712_iommu *mmu = domain_to_mmu(domain);
 	u32 entry = MMMU_PTE_VALID | (pa >> IOMMU_PAGE_SHIFT);
 	u32 align = (u32)(iova | pa | bytes);
-	unsigned int p;
 	unsigned long flags;
+	unsigned int p, p_last, lxp;
+	bool dirty_top = false;
 
 	/* Reject if not entirely within our aperture (should never happen) */
 	bytes *= count;
 	if (iova < mmu->aperture_base || iova + bytes > mmu->aperture_end) {
 		*mapped = 0;
 		return -EINVAL;
 	}
+
+	/* DMA addresses -> page numbers */
 	iova -= mmu->aperture_base;
+	p = iova >> IOMMU_PAGE_SHIFT;
+	p_last = (iova + bytes - 1) >> IOMMU_PAGE_SHIFT;
 
 	/*
 	 * Check we have allocated the required Level-2 tables (in units of
@@ -285,25 +283,33 @@ static int bcm2712_iommu_map(struct iommu_domain *domain, unsigned long iova,
 	 * A failure here will cause the entire map() call to fail.
 	 */
 	spin_lock_irqsave(&mmu->hw_lock, flags);
-	for (p = iova >> TABLES_LXPAGE_SHIFT;
-	     p <= (iova + bytes - 1) >> TABLES_LXPAGE_SHIFT; p++) {
-		if (!mmu->tables[p]) {
-			unsigned int p1, u;
-
-			u = bcm2712_iommu_getpage(mmu, &mmu->tables[p]);
-			if (!u) {
-				spin_unlock_irqrestore(&mmu->hw_lock, flags);
-				*mapped = 0;
-				return -ENOMEM;
-			}
+	for (lxp = (p >> LX_PAGEWORDS_SHIFT);
+	     lxp <= (p_last >> LX_PAGEWORDS_SHIFT); lxp++) {
+		if (!mmu->tables[lxp]) {
+			unsigned int i, u;
+
+			u = bcm2712_iommu_getpage(mmu, &mmu->tables[lxp]);
+			if (!u)
+				break;
 			u |= MMMU_PTE_VALID;
-			for (p1 = p << (PAGE_SHIFT - IOMMU_PAGE_SHIFT);
-			     p1 < (p + 1) << (PAGE_SHIFT - IOMMU_PAGE_SHIFT); p1++) {
-				mmu->top_table[p1] = u++;
+			if (!dirty_top)
+				dma_sync_single_for_cpu(mmu->dev, virt_to_phys(&mmu->top_table),
+							PAGE_SIZE, DMA_TO_DEVICE);
+			for (i = lxp << (PAGE_SHIFT - IOMMU_PAGE_SHIFT);
+			     i < (lxp + 1) << (PAGE_SHIFT - IOMMU_PAGE_SHIFT); i++) {
+				mmu->top_table[i] = u++;
 			}
-			mmu->dirty_top = true;
+			dirty_top = true;
 		}
 	}
+	if (dirty_top)
+		dma_sync_single_for_device(mmu->dev, virt_to_phys(&mmu->top_table),
+					   PAGE_SIZE, DMA_TO_DEVICE);
+	if (lxp <= (p_last >> TABLES_LXPAGE_SHIFT)) {
+		spin_unlock_irqrestore(&mmu->hw_lock, flags);
+		*mapped = 0;
+		return -ENOMEM;
+	}
 
 	/* large page and write enable flags */
 	if (!(align & ((1 << IOMMU_HUGEPAGE_SHIFT) - 1)))
@@ -315,14 +321,25 @@ static int bcm2712_iommu_map(struct iommu_domain *domain, unsigned long iova,
 	if (prot & IOMMU_WRITE)
 		entry |= MMMU_PTE_WRITEABLE;
 
-	/* Now fill in the level-2 tables */
-	for (p = iova >> IOMMU_PAGE_SHIFT;
-	     p < (iova + bytes) >> IOMMU_PAGE_SHIFT; p++) {
+	/*
+	 * Again iterate over table-pages and bring them into CPU ownwership.
+	 * Fill in the required PT entries, then give them back to the device.
+	 */
+	while (p <= p_last) {
 		u32 *tbl = mmu->tables[p >> LX_PAGEWORDS_SHIFT];
 
-		mmu->nmapped_pages += !tbl[p & LX_PAGEWORDS_MASK];
-		tbl[p & LX_PAGEWORDS_MASK] = entry++;
+		dma_sync_single_for_cpu(mmu->dev, virt_to_phys(tbl),
+					PAGE_SIZE, DMA_TO_DEVICE);
+		while (p <= p_last) {
+			mmu->nmapped_pages += !tbl[p & LX_PAGEWORDS_MASK];
+			tbl[p & LX_PAGEWORDS_MASK] = entry++;
+			if (IS_ALIGNED(++p, (1u << LX_PAGEWORDS_SHIFT)))
+				break;
+		}
+		dma_sync_single_for_device(mmu->dev, virt_to_phys(tbl),
+					   PAGE_SIZE, DMA_TO_DEVICE);
 	}
+
 	spin_unlock_irqrestore(&mmu->hw_lock, flags);
 	*mapped = bytes;
 	return 0;
@@ -334,7 +351,7 @@ static size_t bcm2712_iommu_unmap(struct iommu_domain *domain, unsigned long iov
 {
 	struct bcm2712_iommu *mmu = domain_to_mmu(domain);
 	unsigned long flags;
-	unsigned int p;
+	unsigned int p, p_last;
 
 	/* Reject if not entirely within our aperture (should never happen) */
 	bytes *= count;
@@ -345,18 +362,29 @@ static size_t bcm2712_iommu_unmap(struct iommu_domain *domain, unsigned long iov
 	spin_lock_irqsave(&mmu->hw_lock, flags);
 	iommu_iotlb_gather_add_range(gather, iova, bytes);
 
-	/* DMA address -> address relative to region mapped by tables */
+	/* DMA addresses -> page numbers */
 	iova -= mmu->aperture_base;
+	p = iova >> IOMMU_PAGE_SHIFT;
+	p_last = (iova + bytes - 1) >> IOMMU_PAGE_SHIFT;
 
-	/* Clear table entries, this marks the addresses as illegal */
-	for (p = iova >> IOMMU_PAGE_SHIFT;
-	     p < (iova + bytes) >> IOMMU_PAGE_SHIFT;
-	     p++) {
+	/*
+	 * Iterate over tables in Linux-pages and bring them into CPU ownwership.
+	 * Clear the required PT entries, then give them back to the device.
+	 */
+	while (p <= p_last) {
 		u32 *tbl = mmu->tables[p >> LX_PAGEWORDS_SHIFT];
 
-		if (tbl && tbl[p & LX_PAGEWORDS_MASK]) {
-			tbl[p & LX_PAGEWORDS_MASK] = 0;
-			mmu->nmapped_pages--;
+		if (tbl) {
+			dma_sync_single_for_cpu(mmu->dev, virt_to_phys(tbl),
+						PAGE_SIZE, DMA_TO_DEVICE);
+			while (p <= p_last) {
+				mmu->nmapped_pages -= !!tbl[p & LX_PAGEWORDS_MASK];
+				tbl[p & LX_PAGEWORDS_MASK] = 0;
+				if (IS_ALIGNED(++p, (1u << LX_PAGEWORDS_SHIFT)))
+					break;
+			}
+			dma_sync_single_for_device(mmu->dev, virt_to_phys(tbl),
+						   PAGE_SIZE, DMA_TO_DEVICE);
 		}
 	}
 
@@ -379,24 +407,8 @@ static int bcm2712_iommu_sync_range(struct iommu_domain *domain,
 	if (iova_end <= iova)
 		return 0;
 
-	/* Ensure tables are cleaned from CPU cache or write-buffer */
-	spin_lock_irqsave(&mmu->hw_lock, flags);
-	for (i = (iova - mmu->aperture_base) >> TABLES_LXPAGE_SHIFT;
-	     i <= (iova_end - mmu->aperture_base - 1) >> TABLES_LXPAGE_SHIFT; i++) {
-		if (mmu->tables[i]) {
-			dma_sync_single_for_device(mmu->dev,
-						   virt_to_phys(mmu->tables[i]),
-						   PAGE_SIZE, DMA_TO_DEVICE);
-		}
-	}
-	if (mmu->dirty_top) {
-		dma_sync_single_for_device(mmu->dev,
-					   virt_to_phys(mmu->top_table),
-					   PAGE_SIZE, DMA_TO_DEVICE);
-		mmu->dirty_top = false;
-	}
-
 	/* Flush the shared TLB cache */
+	spin_lock_irqsave(&mmu->hw_lock, flags);
 	if (mmu->cache)
 		bcm2712_iommu_cache_flush(mmu->cache);
 
@@ -460,7 +472,7 @@ static void bcm2712_iommu_sync_all(struct iommu_domain *domain)
 
 static phys_addr_t bcm2712_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
 {
-	phys_addr_t addr = (phys_addr_t)(-EINVAL);
+	phys_addr_t addr = 0;
 	struct bcm2712_iommu *mmu = domain_to_mmu(domain);
 
 	if (iova  >= mmu->aperture_base && iova < mmu->aperture_end) {

drivers/iommu/bcm2712-iommu.h

@@ -37,7 +37,6 @@ struct bcm2712_iommu {
 	u32 bigpage_mask;
 	u32 superpage_mask;
 	unsigned int nmapped_pages;
-	bool dirty_top;
 };
 
 struct bcm2712_iommu_domain {