iommu/arm-smmu-v3: Use burst-polling for sync completion

While CMD_SYNC is unlikely to complete immediately such that we never go
round the polling loop, with a lightly-loaded queue it may still do so
long before the delay period is up. If we have no better completion
notifier, use similar logic as we have for SMMUv2 to spin a number of
times before each backoff, so that we have more chance of catching syncs
which complete relatively quickly and avoid delaying unnecessarily.

Signed-off-by: Robin Murphy <[email protected]>
Signed-off-by: Will Deacon <[email protected]>

diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index c148f76dd8e282f1adf4a274fa70043894f1eb60..bfab719190e8690963b2c4323f270a7fd24447a5 100644 (file)
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -419,6 +419,7 @@
 /* High-level queue structures */
 #define ARM_SMMU_POLL_TIMEOUT_US       100
 #define ARM_SMMU_CMDQ_SYNC_TIMEOUT_US  1000000 /* 1s! */
+#define ARM_SMMU_CMDQ_SYNC_SPIN_COUNT  10
 
 #define MSI_IOVA_BASE                  0x8000000
 #define MSI_IOVA_LENGTH                        0x100000
@@ -769,7 +770,7 @@ static void queue_inc_prod(struct arm_smmu_queue *q)
 static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
 {
        ktime_t timeout;
-       unsigned int delay = 1;
+       unsigned int delay = 1, spin_cnt = 0;
 
        /* Wait longer if it's a CMD_SYNC */
        timeout = ktime_add_us(ktime_get(), sync ?
@@ -782,10 +783,13 @@ static int queue_poll_cons(struct arm_smmu_queue *q, bool sync, bool wfe)
 
                if (wfe) {
                        wfe();
-               } else {
+               } else if (++spin_cnt < ARM_SMMU_CMDQ_SYNC_SPIN_COUNT) {
                        cpu_relax();
+                       continue;
+               } else {
                        udelay(delay);
                        delay *= 2;
+                       spin_cnt = 0;
                }
        }