powerpc/iommu: Reduce spinlock coverage in iommu_alloc and iommu_free

We currently hold the IOMMU spinlock around tce_build and tce_flush.
This causes our spinlock hold times to be much higher than required
and can impact multiqueue adapters.

This patch moves tce_build and tce_flush outside of the lock in
iommu_alloc, and tce_flush outside of the lock in iommu_free.

Some performance numbers were obtained with a Chelsio T3 adapter on
two POWER7 boxes, running a 100 session TCP round robin test.

Performance improved 32% with this patch applied.

Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c
index 359f078571c7e5c9da9a6b90e5a53e0feec78b06..9c8967fa1e63474cf6d0aa905a36f945d5c389ab 100644 (file)
--- a/arch/powerpc/kernel/iommu.c
+++ b/arch/powerpc/kernel/iommu.c
@@ -170,13 +170,11 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
        int build_fail;
 
        spin_lock_irqsave(&(tbl->it_lock), flags);
-
        entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order);
+       spin_unlock_irqrestore(&(tbl->it_lock), flags);
 
-       if (unlikely(entry == DMA_ERROR_CODE)) {
-               spin_unlock_irqrestore(&(tbl->it_lock), flags);
+       if (unlikely(entry == DMA_ERROR_CODE))
                return DMA_ERROR_CODE;
-       }
 
        entry += tbl->it_offset;        /* Offset into real TCE table */
        ret = entry << IOMMU_PAGE_SHIFT;        /* Set the return dma address */
@@ -192,9 +190,10 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
         * not altered.
         */
        if (unlikely(build_fail)) {
+               spin_lock_irqsave(&(tbl->it_lock), flags);
                __iommu_free(tbl, ret, npages);
-
                spin_unlock_irqrestore(&(tbl->it_lock), flags);
+
                return DMA_ERROR_CODE;
        }
 
@@ -202,8 +201,6 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
        if (ppc_md.tce_flush)
                ppc_md.tce_flush(tbl);
 
-       spin_unlock_irqrestore(&(tbl->it_lock), flags);
-
        /* Make sure updates are seen by hardware */
        mb();
 
@@ -244,8 +241,8 @@ static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
        unsigned long flags;
 
        spin_lock_irqsave(&(tbl->it_lock), flags);
-
        __iommu_free(tbl, dma_addr, npages);
+       spin_unlock_irqrestore(&(tbl->it_lock), flags);
 
        /* Make sure TLB cache is flushed if the HW needs it. We do
         * not do an mb() here on purpose, it is not needed on any of
@@ -253,8 +250,6 @@ static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
         */
        if (ppc_md.tce_flush)
                ppc_md.tce_flush(tbl);
-
-       spin_unlock_irqrestore(&(tbl->it_lock), flags);
 }
 
 int iommu_map_sg(struct device *dev, struct iommu_table *tbl,