blk-mq: switch ctx pending map to the sparser blk_align_bitmap

Each hardware queue has a bitmap of software queues with pending
requests. When new IO is queued on a software queue, the bit is
set, and when IO is pruned on a hardware queue run, the bit is
cleared. This causes a lot of traffic. Switch this from the regular
BITS_PER_LONG bitmap to a sparser layout, similarly to what was
done for blk-mq tagging.

20% performance increase was observed for single threaded IO, and
about 15% performanc increase on multiple threads driving the
same device.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 526feee31bffdc9d663438bb84a757793c38fbd6..e862c4408427eff676e4ddf596e9fa3d31f3a758 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -56,21 +56,40 @@ static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
 {
        unsigned int i;
 
-       for (i = 0; i < hctx->nr_ctx_map; i++)
-               if (hctx->ctx_map[i])
+       for (i = 0; i < hctx->ctx_map.map_size; i++)
+               if (hctx->ctx_map.map[i].word)
                        return true;
 
        return false;
 }
 
+static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
+                                             struct blk_mq_ctx *ctx)
+{
+       return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
+}
+
+#define CTX_TO_BIT(hctx, ctx)  \
+       ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
+
 /*
  * Mark this ctx as having pending work in this hardware queue
  */
 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
                                     struct blk_mq_ctx *ctx)
 {
-       if (!test_bit(ctx->index_hw, hctx->ctx_map))
-               set_bit(ctx->index_hw, hctx->ctx_map);
+       struct blk_align_bitmap *bm = get_bm(hctx, ctx);
+
+       if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
+               set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
+}
+
+static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
+                                     struct blk_mq_ctx *ctx)
+{
+       struct blk_align_bitmap *bm = get_bm(hctx, ctx);
+
+       clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
 }
 
 static struct request *__blk_mq_alloc_request(struct blk_mq_hw_ctx *hctx,
@@ -614,6 +633,40 @@ static bool blk_mq_attempt_merge(struct request_queue *q,
        return false;
 }
 
+/*
+ * Process software queues that have been marked busy, splicing them
+ * to the for-dispatch
+ */
+static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
+{
+       struct blk_mq_ctx *ctx;
+       int i;
+
+       for (i = 0; i < hctx->ctx_map.map_size; i++) {
+               struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
+               unsigned int off, bit;
+
+               if (!bm->word)
+                       continue;
+
+               bit = 0;
+               off = i * hctx->ctx_map.bits_per_word;
+               do {
+                       bit = find_next_bit(&bm->word, bm->depth, bit);
+                       if (bit >= bm->depth)
+                               break;
+
+                       ctx = hctx->ctxs[bit + off];
+                       clear_bit(bit, &bm->word);
+                       spin_lock(&ctx->lock);
+                       list_splice_tail_init(&ctx->rq_list, list);
+                       spin_unlock(&ctx->lock);
+
+                       bit++;
+               } while (1);
+       }
+}
+
 /*
  * Run this hardware queue, pulling any software queues mapped to it in.
  * Note that this function currently has various problems around ordering
@@ -623,10 +676,9 @@ static bool blk_mq_attempt_merge(struct request_queue *q,
 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 {
        struct request_queue *q = hctx->queue;
-       struct blk_mq_ctx *ctx;
        struct request *rq;
        LIST_HEAD(rq_list);
-       int bit, queued;
+       int queued;
 
        WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 
@@ -638,14 +690,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
        /*
         * Touch any software queue that has pending entries.
         */
-       for_each_set_bit(bit, hctx->ctx_map, hctx->nr_ctx) {
-               clear_bit(bit, hctx->ctx_map);
-               ctx = hctx->ctxs[bit];
-
-               spin_lock(&ctx->lock);
-               list_splice_tail_init(&ctx->rq_list, &rq_list);
-               spin_unlock(&ctx->lock);
-       }
+       flush_busy_ctxs(hctx, &rq_list);
 
        /*
         * If we have previous entries on our dispatch list, grab them
@@ -658,14 +703,10 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
                spin_unlock(&hctx->lock);
        }
 
-       /*
-        * Delete and return all entries from our dispatch list
-        */
-       queued = 0;
-
        /*
         * Now process all the entries, sending them to the driver.
         */
+       queued = 0;
        while (!list_empty(&rq_list)) {
                int ret;
 
@@ -1158,7 +1199,7 @@ static void blk_mq_hctx_notify(void *data, unsigned long action,
        spin_lock(&ctx->lock);
        if (!list_empty(&ctx->rq_list)) {
                list_splice_init(&ctx->rq_list, &tmp);
-               clear_bit(ctx->index_hw, hctx->ctx_map);
+               blk_mq_hctx_clear_pending(hctx, ctx);
        }
        spin_unlock(&ctx->lock);
 
@@ -1298,6 +1339,34 @@ fail:
        return NULL;
 }
 
+static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
+{
+       kfree(bitmap->map);
+}
+
+static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
+{
+       unsigned int bpw = 8, total, num_maps, i;
+
+       bitmap->bits_per_word = bpw;
+
+       num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
+       bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
+                                       GFP_KERNEL, node);
+       if (!bitmap->map)
+               return -ENOMEM;
+
+       bitmap->map_size = num_maps;
+
+       total = nr_cpu_ids;
+       for (i = 0; i < num_maps; i++) {
+               bitmap->map[i].depth = min(total, bitmap->bits_per_word);
+               total -= bitmap->map[i].depth;
+       }
+
+       return 0;
+}
+
 static int blk_mq_init_hw_queues(struct request_queue *q,
                struct blk_mq_tag_set *set)
 {
@@ -1308,7 +1377,6 @@ static int blk_mq_init_hw_queues(struct request_queue *q,
         * Initialize hardware queues
         */
        queue_for_each_hw_ctx(q, hctx, i) {
-               unsigned int num_maps;
                int node;
 
                node = hctx->numa_node;
@@ -1339,13 +1407,9 @@ static int blk_mq_init_hw_queues(struct request_queue *q,
                if (!hctx->ctxs)
                        break;
 
-               num_maps = ALIGN(nr_cpu_ids, BITS_PER_LONG) / BITS_PER_LONG;
-               hctx->ctx_map = kzalloc_node(num_maps * sizeof(unsigned long),
-                                               GFP_KERNEL, node);
-               if (!hctx->ctx_map)
+               if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
                        break;
 
-               hctx->nr_ctx_map = num_maps;
                hctx->nr_ctx = 0;
 
                if (set->ops->init_hctx &&
@@ -1368,7 +1432,7 @@ static int blk_mq_init_hw_queues(struct request_queue *q,
 
                blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
                kfree(hctx->ctxs);
-               kfree(hctx->ctx_map);
+               blk_mq_free_bitmap(&hctx->ctx_map);
        }
 
        return 1;
@@ -1542,7 +1606,6 @@ void blk_mq_free_queue(struct request_queue *q)
        int i;
 
        queue_for_each_hw_ctx(q, hctx, i) {
-               kfree(hctx->ctx_map);
                kfree(hctx->ctxs);
                blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
                if (q->mq_ops->exit_hctx)

diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index f83d15f6e1c1846c88f76f60dd471b39c5c1bfa4..952e558ee598ac0b643d3198afd7f7e8f83ea6da 100644 (file)
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -11,6 +11,12 @@ struct blk_mq_cpu_notifier {
        void (*notify)(void *data, unsigned long action, unsigned int cpu);
 };
 
+struct blk_mq_ctxmap {
+       unsigned int map_size;
+       unsigned int bits_per_word;
+       struct blk_align_bitmap *map;
+};
+
 struct blk_mq_hw_ctx {
        struct {
                spinlock_t              lock;
@@ -31,8 +37,8 @@ struct blk_mq_hw_ctx {
 
        void                    *driver_data;
 
-       unsigned int            nr_ctx_map;
-       unsigned long           *ctx_map;
+       struct blk_mq_ctxmap    ctx_map;
+
        unsigned int            nr_ctx;
        struct blk_mq_ctx       **ctxs;