blk-mq-sched: Allocate sched reserved tags as specified in the original queue tagset

Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Modified by me to also check at driver tag allocation time if the
original request was reserved, so we can be sure to allocate a
properly reserved tag at that point in time, too.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 98c7b061781e55f0176bfc24c6345c0c0611f1f3..46ca965fff5cbf3ed6e0b5bc5f43462cdcade606 100644 (file)
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -454,7 +454,8 @@ int blk_mq_sched_setup(struct request_queue *q)
         */
        ret = 0;
        queue_for_each_hw_ctx(q, hctx, i) {
-               hctx->sched_tags = blk_mq_alloc_rq_map(set, i, q->nr_requests, 0);
+               hctx->sched_tags = blk_mq_alloc_rq_map(set, i,
+                               q->nr_requests, set->reserved_tags);
                if (!hctx->sched_tags) {
                        ret = -ENOMEM;
                        break;

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 54c84363c1b2385899472d77cb4fc93c81a59643..e48bc2c72615de016f013a2e98ea72cd49713a04 100644 (file)
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -181,7 +181,7 @@ found_tag:
 void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags,
                    struct blk_mq_ctx *ctx, unsigned int tag)
 {
-       if (tag >= tags->nr_reserved_tags) {
+       if (!blk_mq_tag_is_reserved(tags, tag)) {
                const int real_tag = tag - tags->nr_reserved_tags;
 
                BUG_ON(real_tag >= tags->nr_tags);

diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 63497423c5cd32fb50542cf626590693c5f929e7..5cb51e53cc0353cbba8e44ae808e6d2f9f9257a7 100644 (file)
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -85,4 +85,10 @@ static inline void blk_mq_tag_set_rq(struct blk_mq_hw_ctx *hctx,
        hctx->tags->rqs[tag] = rq;
 }
 
+static inline bool blk_mq_tag_is_reserved(struct blk_mq_tags *tags,
+                                         unsigned int tag)
+{
+       return tag < tags->nr_reserved_tags;
+}
+
 #endif

diff --git a/block/blk-mq.c b/block/blk-mq.c
index d84c66fb37b78a959b1de459907715cc65026a38..4df5fb42c74fe50922d402b9e6da23788867c2cb 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -852,6 +852,9 @@ done:
                return true;
        }
 
+       if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
+               data.flags |= BLK_MQ_REQ_RESERVED;
+
        rq->tag = blk_mq_get_tag(&data);
        if (rq->tag >= 0) {
                if (blk_mq_tag_busy(data.hctx)) {