NVMe: Fix nvme get/put queue semantics

The routines to get and lock nvme queues required the caller to "put"
or "unlock" them even if getting one returned NULL. This patch fixes that.

Signed-off-by: Keith Busch <keith.busch@intel.com>
Signed-off-by: Matthew Wilcox <matthew.r.wilcox@intel.com>

diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c
index 12c57eb7c915097ee0673c31b67864191bea4527..29a3e85873b561b458c53638e1bf15fedcf1e4f1 100644 (file)
--- a/drivers/block/nvme-core.c
+++ b/drivers/block/nvme-core.c
@@ -285,9 +285,17 @@ static struct nvme_queue *raw_nvmeq(struct nvme_dev *dev, int qid)
 
 static struct nvme_queue *get_nvmeq(struct nvme_dev *dev) __acquires(RCU)
 {
+       struct nvme_queue *nvmeq;
        unsigned queue_id = get_cpu_var(*dev->io_queue);
+
        rcu_read_lock();
-       return rcu_dereference(dev->queues[queue_id]);
+       nvmeq = rcu_dereference(dev->queues[queue_id]);
+       if (nvmeq)
+               return nvmeq;
+
+       rcu_read_unlock();
+       put_cpu_var(*dev->io_queue);
+       return NULL;
 }
 
 static void put_nvmeq(struct nvme_queue *nvmeq) __releases(RCU)
@@ -299,8 +307,15 @@ static void put_nvmeq(struct nvme_queue *nvmeq) __releases(RCU)
 static struct nvme_queue *lock_nvmeq(struct nvme_dev *dev, int q_idx)
                                                        __acquires(RCU)
 {
+       struct nvme_queue *nvmeq;
+
        rcu_read_lock();
-       return rcu_dereference(dev->queues[q_idx]);
+       nvmeq = rcu_dereference(dev->queues[q_idx]);
+       if (nvmeq)
+               return nvmeq;
+
+       rcu_read_unlock();
+       return NULL;
 }
 
 static void unlock_nvmeq(struct nvme_queue *nvmeq) __releases(RCU)
@@ -809,7 +824,6 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)
        int result = -EBUSY;
 
        if (!nvmeq) {
-               put_nvmeq(NULL);
                bio_endio(bio, -EIO);
                return;
        }
@@ -884,10 +898,8 @@ static int nvme_submit_sync_cmd(struct nvme_dev *dev, int q_idx,
        struct nvme_queue *nvmeq;
 
        nvmeq = lock_nvmeq(dev, q_idx);
-       if (!nvmeq) {
-               unlock_nvmeq(nvmeq);
+       if (!nvmeq)
                return -ENODEV;
-       }
 
        cmdinfo.task = current;
        cmdinfo.status = -EINTR;
@@ -912,9 +924,10 @@ static int nvme_submit_sync_cmd(struct nvme_dev *dev, int q_idx,
 
        if (cmdinfo.status == -EINTR) {
                nvmeq = lock_nvmeq(dev, q_idx);
-               if (nvmeq)
+               if (nvmeq) {
                        nvme_abort_command(nvmeq, cmdid);
-               unlock_nvmeq(nvmeq);
+                       unlock_nvmeq(nvmeq);
+               }
                return -EINTR;
        }