lightnvm: pblk: simplify work_queue mempool

In pblk, we have a mempool to allocate a generic structure that we
pass along workqueues. This is heavily used in the GC path in order
to have enough inflight reads and fully utilize the GC bandwidth.

However, the current GC path copies data to the host memory and puts it
back into the write buffer. This requires a vmalloc allocation for the
data and a memory copy. Thus, guaranteeing the allocation by using a
mempool for the structure in itself does not give us much. Until we
implement support for vector copy to avoid moving data through the host,
just allocate the workqueue structure using kmalloc.

This allows us to have a much smaller mempool.

Reported-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Javier González <javier@cnexlabs.com>
Signed-off-by: Matias Bjørling <m@bjorling.me>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/drivers/lightnvm/pblk-core.c b/drivers/lightnvm/pblk-core.c
index f5fbb9a46784cc380bed67310bc515dbbdf932a2..b92532211866d3610b7858a9e143d424ada3e4c4 100644 (file)
--- a/drivers/lightnvm/pblk-core.c
+++ b/drivers/lightnvm/pblk-core.c
@@ -33,7 +33,8 @@ static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line,
                pr_err("pblk: attempted to erase bb: line:%d, pos:%d\n",
                                                        line->id, pos);
 
-       pblk_line_run_ws(pblk, NULL, ppa, pblk_line_mark_bb, pblk->bb_wq);
+       pblk_gen_run_ws(pblk, NULL, ppa, pblk_line_mark_bb,
+                                               GFP_ATOMIC, pblk->bb_wq);
 }
 
 static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd)
@@ -1623,7 +1624,7 @@ void pblk_line_close_ws(struct work_struct *work)
        struct pblk_line *line = line_ws->line;
 
        pblk_line_close(pblk, line);
-       mempool_free(line_ws, pblk->line_ws_pool);
+       mempool_free(line_ws, pblk->gen_ws_pool);
 }
 
 void pblk_line_mark_bb(struct work_struct *work)
@@ -1648,16 +1649,16 @@ void pblk_line_mark_bb(struct work_struct *work)
        }
 
        kfree(ppa);
-       mempool_free(line_ws, pblk->line_ws_pool);
+       mempool_free(line_ws, pblk->gen_ws_pool);
 }
 
-void pblk_line_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
-                     void (*work)(struct work_struct *),
+void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
+                     void (*work)(struct work_struct *), gfp_t gfp_mask,
                      struct workqueue_struct *wq)
 {
        struct pblk_line_ws *line_ws;
 
-       line_ws = mempool_alloc(pblk->line_ws_pool, GFP_ATOMIC);
+       line_ws = mempool_alloc(pblk->gen_ws_pool, gfp_mask);
        if (!line_ws)
                return;
 

diff --git a/drivers/lightnvm/pblk-gc.c b/drivers/lightnvm/pblk-gc.c
index 6090d28f7995a51f4d5972bf965457f1388874f7..f163829ecca826afc0397784b43ea4cfd71b48f8 100644 (file)
--- a/drivers/lightnvm/pblk-gc.c
+++ b/drivers/lightnvm/pblk-gc.c
@@ -136,12 +136,12 @@ static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
 
 static void pblk_gc_line_ws(struct work_struct *work)
 {
-       struct pblk_line_ws *line_rq_ws = container_of(work,
+       struct pblk_line_ws *gc_rq_ws = container_of(work,
                                                struct pblk_line_ws, ws);
-       struct pblk *pblk = line_rq_ws->pblk;
+       struct pblk *pblk = gc_rq_ws->pblk;
        struct pblk_gc *gc = &pblk->gc;
-       struct pblk_line *line = line_rq_ws->line;
-       struct pblk_gc_rq *gc_rq = line_rq_ws->priv;
+       struct pblk_line *line = gc_rq_ws->line;
+       struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
 
        up(&gc->gc_sem);
 
@@ -151,7 +151,7 @@ static void pblk_gc_line_ws(struct work_struct *work)
                                                gc_rq->nr_secs);
        }
 
-       mempool_free(line_rq_ws, pblk->line_ws_pool);
+       kfree(gc_rq_ws);
 }
 
 static void pblk_gc_line_prepare_ws(struct work_struct *work)
@@ -164,7 +164,7 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
        struct pblk_line_meta *lm = &pblk->lm;
        struct pblk_gc *gc = &pblk->gc;
        struct line_emeta *emeta_buf;
-       struct pblk_line_ws *line_rq_ws;
+       struct pblk_line_ws *gc_rq_ws;
        struct pblk_gc_rq *gc_rq;
        __le64 *lba_list;
        int sec_left, nr_secs, bit;
@@ -223,19 +223,19 @@ next_rq:
        gc_rq->nr_secs = nr_secs;
        gc_rq->line = line;
 
-       line_rq_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL);
-       if (!line_rq_ws)
+       gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
+       if (!gc_rq_ws)
                goto fail_free_gc_rq;
 
-       line_rq_ws->pblk = pblk;
-       line_rq_ws->line = line;
-       line_rq_ws->priv = gc_rq;
+       gc_rq_ws->pblk = pblk;
+       gc_rq_ws->line = line;
+       gc_rq_ws->priv = gc_rq;
 
        down(&gc->gc_sem);
        kref_get(&line->ref);
 
-       INIT_WORK(&line_rq_ws->ws, pblk_gc_line_ws);
-       queue_work(gc->gc_line_reader_wq, &line_rq_ws->ws);
+       INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
+       queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);
 
        sec_left -= nr_secs;
        if (sec_left > 0)
@@ -243,7 +243,7 @@ next_rq:
 
 out:
        pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
-       mempool_free(line_ws, pblk->line_ws_pool);
+       kfree(line_ws);
 
        kref_put(&line->ref, pblk_line_put);
        atomic_dec(&gc->inflight_gc);
@@ -256,7 +256,7 @@ fail_free_emeta:
        pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
        pblk_put_line_back(pblk, line);
        kref_put(&line->ref, pblk_line_put);
-       mempool_free(line_ws, pblk->line_ws_pool);
+       kfree(line_ws);
        atomic_dec(&gc->inflight_gc);
 
        pr_err("pblk: Failed to GC line %d\n", line->id);
@@ -269,7 +269,7 @@ static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)
 
        pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id);
 
-       line_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL);
+       line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
        if (!line_ws)
                return -ENOMEM;
 

diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c
index 7b1f29c7133813eb64e309dc9e57c348af37602f..34055225358023587798414d11968199952b3bf4 100644 (file)
--- a/drivers/lightnvm/pblk-init.c
+++ b/drivers/lightnvm/pblk-init.c
@@ -20,7 +20,7 @@
 
 #include "pblk.h"
 
-static struct kmem_cache *pblk_blk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
+static struct kmem_cache *pblk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
                                *pblk_w_rq_cache, *pblk_line_meta_cache;
 static DECLARE_RWSEM(pblk_lock);
 struct bio_set *pblk_bio_set;
@@ -184,9 +184,9 @@ static int pblk_init_global_caches(struct pblk *pblk)
        char cache_name[PBLK_CACHE_NAME_LEN];
 
        down_write(&pblk_lock);
-       pblk_blk_ws_cache = kmem_cache_create("pblk_blk_ws",
+       pblk_ws_cache = kmem_cache_create("pblk_blk_ws",
                                sizeof(struct pblk_line_ws), 0, 0, NULL);
-       if (!pblk_blk_ws_cache) {
+       if (!pblk_ws_cache) {
                up_write(&pblk_lock);
                return -ENOMEM;
        }
@@ -194,7 +194,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
        pblk_rec_cache = kmem_cache_create("pblk_rec",
                                sizeof(struct pblk_rec_ctx), 0, 0, NULL);
        if (!pblk_rec_cache) {
-               kmem_cache_destroy(pblk_blk_ws_cache);
+               kmem_cache_destroy(pblk_ws_cache);
                up_write(&pblk_lock);
                return -ENOMEM;
        }
@@ -202,7 +202,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
        pblk_g_rq_cache = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
                                0, 0, NULL);
        if (!pblk_g_rq_cache) {
-               kmem_cache_destroy(pblk_blk_ws_cache);
+               kmem_cache_destroy(pblk_ws_cache);
                kmem_cache_destroy(pblk_rec_cache);
                up_write(&pblk_lock);
                return -ENOMEM;
@@ -211,7 +211,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
        pblk_w_rq_cache = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
                                0, 0, NULL);
        if (!pblk_w_rq_cache) {
-               kmem_cache_destroy(pblk_blk_ws_cache);
+               kmem_cache_destroy(pblk_ws_cache);
                kmem_cache_destroy(pblk_rec_cache);
                kmem_cache_destroy(pblk_g_rq_cache);
                up_write(&pblk_lock);
@@ -223,7 +223,7 @@ static int pblk_init_global_caches(struct pblk *pblk)
        pblk_line_meta_cache = kmem_cache_create(cache_name,
                                pblk->lm.sec_bitmap_len, 0, 0, NULL);
        if (!pblk_line_meta_cache) {
-               kmem_cache_destroy(pblk_blk_ws_cache);
+               kmem_cache_destroy(pblk_ws_cache);
                kmem_cache_destroy(pblk_rec_cache);
                kmem_cache_destroy(pblk_g_rq_cache);
                kmem_cache_destroy(pblk_w_rq_cache);
@@ -246,20 +246,20 @@ static int pblk_core_init(struct pblk *pblk)
        if (pblk_init_global_caches(pblk))
                return -ENOMEM;
 
-       /* internal bios can be at most the sectors signaled by the device. */
+       /* Internal bios can be at most the sectors signaled by the device. */
        pblk->page_bio_pool = mempool_create_page_pool(nvm_max_phys_sects(dev),
                                                                        0);
        if (!pblk->page_bio_pool)
                return -ENOMEM;
 
-       pblk->line_ws_pool = mempool_create_slab_pool(PBLK_WS_POOL_SIZE,
-                                                       pblk_blk_ws_cache);
-       if (!pblk->line_ws_pool)
+       pblk->gen_ws_pool = mempool_create_slab_pool(PBLK_GEN_WS_POOL_SIZE,
+                                                       pblk_ws_cache);
+       if (!pblk->gen_ws_pool)
                goto free_page_bio_pool;
 
        pblk->rec_pool = mempool_create_slab_pool(geo->nr_luns, pblk_rec_cache);
        if (!pblk->rec_pool)
-               goto free_blk_ws_pool;
+               goto free_gen_ws_pool;
 
        pblk->g_rq_pool = mempool_create_slab_pool(PBLK_READ_REQ_POOL_SIZE,
                                                        pblk_g_rq_cache);
@@ -308,8 +308,8 @@ free_g_rq_pool:
        mempool_destroy(pblk->g_rq_pool);
 free_rec_pool:
        mempool_destroy(pblk->rec_pool);
-free_blk_ws_pool:
-       mempool_destroy(pblk->line_ws_pool);
+free_gen_ws_pool:
+       mempool_destroy(pblk->gen_ws_pool);
 free_page_bio_pool:
        mempool_destroy(pblk->page_bio_pool);
        return -ENOMEM;
@@ -324,13 +324,13 @@ static void pblk_core_free(struct pblk *pblk)
                destroy_workqueue(pblk->bb_wq);
 
        mempool_destroy(pblk->page_bio_pool);
-       mempool_destroy(pblk->line_ws_pool);
+       mempool_destroy(pblk->gen_ws_pool);
        mempool_destroy(pblk->rec_pool);
        mempool_destroy(pblk->g_rq_pool);
        mempool_destroy(pblk->w_rq_pool);
        mempool_destroy(pblk->line_meta_pool);
 
-       kmem_cache_destroy(pblk_blk_ws_cache);
+       kmem_cache_destroy(pblk_ws_cache);
        kmem_cache_destroy(pblk_rec_cache);
        kmem_cache_destroy(pblk_g_rq_cache);
        kmem_cache_destroy(pblk_w_rq_cache);

diff --git a/drivers/lightnvm/pblk-write.c b/drivers/lightnvm/pblk-write.c
index d82ca8bd83908b037ee28b28e554da0fa5b4f7d5..c73b17bca06b0cb7abf03a0534223d6ec9b91892 100644 (file)
--- a/drivers/lightnvm/pblk-write.c
+++ b/drivers/lightnvm/pblk-write.c
@@ -198,8 +198,8 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd)
 
        sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);
        if (sync == emeta->nr_entries)
-               pblk_line_run_ws(pblk, line, NULL, pblk_line_close_ws,
-                                                               pblk->close_wq);
+               pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws,
+                                               GFP_ATOMIC, pblk->close_wq);
 
        bio_put(rqd->bio);
        nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);

diff --git a/drivers/lightnvm/pblk.h b/drivers/lightnvm/pblk.h
index 229f6020ad8ad8f539391454d66e26754650c7ff..efaa781abb06cd2e5de9c26ef2a0f653a048443f 100644 (file)
--- a/drivers/lightnvm/pblk.h
+++ b/drivers/lightnvm/pblk.h
@@ -40,7 +40,6 @@
 #define PBLK_MAX_REQ_ADDRS (64)
 #define PBLK_MAX_REQ_ADDRS_PW (6)
 
-#define PBLK_WS_POOL_SIZE (128)
 #define PBLK_META_POOL_SIZE (128)
 #define PBLK_READ_REQ_POOL_SIZE (1024)
 
@@ -61,6 +60,8 @@
 
 #define ERASE 2 /* READ = 0, WRITE = 1 */
 
+#define PBLK_GEN_WS_POOL_SIZE (2)
+
 enum {
        /* IO Types */
        PBLK_IOTYPE_USER        = 1 << 0,
@@ -621,7 +622,7 @@ struct pblk {
        struct list_head compl_list;
 
        mempool_t *page_bio_pool;
-       mempool_t *line_ws_pool;
+       mempool_t *gen_ws_pool;
        mempool_t *rec_pool;
        mempool_t *g_rq_pool;
        mempool_t *w_rq_pool;
@@ -725,9 +726,9 @@ void pblk_line_close_meta_sync(struct pblk *pblk);
 void pblk_line_close_ws(struct work_struct *work);
 void pblk_pipeline_stop(struct pblk *pblk);
 void pblk_line_mark_bb(struct work_struct *work);
-void pblk_line_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
-                     void (*work)(struct work_struct *),
-                     struct workqueue_struct *wq);
+void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
+                    void (*work)(struct work_struct *), gfp_t gfp_mask,
+                    struct workqueue_struct *wq);
 u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line);
 int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line);
 int pblk_line_read_emeta(struct pblk *pblk, struct pblk_line *line,