vhost: try avoiding avail index access when getting descriptor

If last avail idx is not equal to cached avail idx, we're sure there's
still available buffers in the virtqueue so there's no need to re-read
avail idx. So let's skip this to avoid unnecessary userspace memory
access and memory barrier. Pktgen test show about 3% improvement on rx
pps.

Signed-off-by: Jason Wang <jasowang@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 8f99fe08de02e7b48725a99d682055c03056b82a..1f7e4e4e6f8efee40262024c493c225e197d2ae8 100644 (file)
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1930,25 +1930,32 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
 
        /* Check it isn't doing very strange things with descriptor numbers. */
        last_avail_idx = vq->last_avail_idx;
-       if (unlikely(vhost_get_user(vq, avail_idx, &vq->avail->idx))) {
-               vq_err(vq, "Failed to access avail idx at %p\n",
-                      &vq->avail->idx);
-               return -EFAULT;
-       }
-       vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
 
-       if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
-               vq_err(vq, "Guest moved used index from %u to %u",
-                      last_avail_idx, vq->avail_idx);
-               return -EFAULT;
-       }
+       if (vq->avail_idx == vq->last_avail_idx) {
+               if (unlikely(vhost_get_user(vq, avail_idx, &vq->avail->idx))) {
+                       vq_err(vq, "Failed to access avail idx at %p\n",
+                               &vq->avail->idx);
+                       return -EFAULT;
+               }
+               vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
 
-       /* If there's nothing new since last we looked, return invalid. */
-       if (vq->avail_idx == last_avail_idx)
-               return vq->num;
+               if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
+                       vq_err(vq, "Guest moved used index from %u to %u",
+                               last_avail_idx, vq->avail_idx);
+                       return -EFAULT;
+               }
+
+               /* If there's nothing new since last we looked, return
+                * invalid.
+                */
+               if (vq->avail_idx == last_avail_idx)
+                       return vq->num;
 
-       /* Only get avail ring entries after they have been exposed by guest. */
-       smp_rmb();
+               /* Only get avail ring entries after they have been
+                * exposed by guest.
+                */
+               smp_rmb();
+       }
 
        /* Grab the next descriptor number they're advertising, and increment
         * the index we've seen. */