nvmet: synchronize sqhd update

In testing target io in read write mix, we did indeed get into cases where
sqhd didn't update properly and slowly missed enough updates to shutdown
the queue.

Protect the updating sqhd by using cmpxchg, and for that turn the sqhd
field into a u32 so that cmpxchg works on it for all architectures.

Signed-off-by: James Smart <james.smart@broadcom.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>

diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c
index 1b208beeef5097c6e4a2bd38e112543b8818af19..645ba7eee35db7a66a0249d39c7adba514173229 100644 (file)
--- a/drivers/nvme/target/core.c
+++ b/drivers/nvme/target/core.c
@@ -387,12 +387,21 @@ struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid)
 
 static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
 {
+       u32 old_sqhd, new_sqhd;
+       u16 sqhd;
+
        if (status)
                nvmet_set_status(req, status);
 
-       if (req->sq->size)
-               req->sq->sqhd = (req->sq->sqhd + 1) % req->sq->size;
-       req->rsp->sq_head = cpu_to_le16(req->sq->sqhd);
+       if (req->sq->size) {
+               do {
+                       old_sqhd = req->sq->sqhd;
+                       new_sqhd = (old_sqhd + 1) % req->sq->size;
+               } while (cmpxchg(&req->sq->sqhd, old_sqhd, new_sqhd) !=
+                                       old_sqhd);
+       }
+       sqhd = req->sq->sqhd & 0x0000FFFF;
+       req->rsp->sq_head = cpu_to_le16(sqhd);
        req->rsp->sq_id = cpu_to_le16(req->sq->qid);
        req->rsp->command_id = req->cmd->common.command_id;
 

diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 7b8e20adf76029f293569688e8a5c71013c6393d..87e429bfcd8a0c918f2aae018c247bb0014d3d0b 100644 (file)
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -74,7 +74,7 @@ struct nvmet_sq {
        struct percpu_ref       ref;
        u16                     qid;
        u16                     size;
-       u16                     sqhd;
+       u32                     sqhd;
        struct completion       free_done;
        struct completion       confirm_done;
 };