staging: zsmalloc: add mapping modes

This patch improves mapping performance in zsmalloc by getting
usage information from the user in the form of a "mapping mode"
and using it to avoid unnecessary copying for objects that span
pages.

Signed-off-by: Seth Jennings <sjenning@linux.vnet.ibm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/drivers/staging/zcache/zcache-main.c b/drivers/staging/zcache/zcache-main.c
index 2b5c73c89c333bf0774f8181473239a988e10962..c214977b4ab48adec84fc260df0116b12b498aa7 100644 (file)
--- a/drivers/staging/zcache/zcache-main.c
+++ b/drivers/staging/zcache/zcache-main.c
@@ -716,7 +716,7 @@ static unsigned long zv_create(struct zs_pool *pool, uint32_t pool_id,
                goto out;
        atomic_inc(&zv_curr_dist_counts[chunks]);
        atomic_inc(&zv_cumul_dist_counts[chunks]);
-       zv = zs_map_object(pool, handle);
+       zv = zs_map_object(pool, handle, ZS_MM_WO);
        zv->index = index;
        zv->oid = *oid;
        zv->pool_id = pool_id;
@@ -735,7 +735,7 @@ static void zv_free(struct zs_pool *pool, unsigned long handle)
        uint16_t size;
        int chunks;
 
-       zv = zs_map_object(pool, handle);
+       zv = zs_map_object(pool, handle, ZS_MM_RW);
        ASSERT_SENTINEL(zv, ZVH);
        size = zv->size + sizeof(struct zv_hdr);
        INVERT_SENTINEL(zv, ZVH);
@@ -757,7 +757,7 @@ static void zv_decompress(struct page *page, unsigned long handle)
        int ret;
        struct zv_hdr *zv;
 
-       zv = zs_map_object(zcache_host.zspool, handle);
+       zv = zs_map_object(zcache_host.zspool, handle, ZS_MM_RO);
        BUG_ON(zv->size == 0);
        ASSERT_SENTINEL(zv, ZVH);
        to_va = kmap_atomic(page);

diff --git a/drivers/staging/zram/zram_drv.c b/drivers/staging/zram/zram_drv.c
index 706cb625aae90f6652a3ef744843e3500fc07e49..653b074035f7b165d04d1587d428097ecfac780a 100644 (file)
--- a/drivers/staging/zram/zram_drv.c
+++ b/drivers/staging/zram/zram_drv.c
@@ -220,7 +220,8 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
                uncmem = user_mem;
        clen = PAGE_SIZE;
 
-       cmem = zs_map_object(zram->mem_pool, zram->table[index].handle);
+       cmem = zs_map_object(zram->mem_pool, zram->table[index].handle,
+                               ZS_MM_RO);
 
        ret = lzo1x_decompress_safe(cmem, zram->table[index].size,
                                    uncmem, &clen);
@@ -258,7 +259,7 @@ static int zram_read_before_write(struct zram *zram, char *mem, u32 index)
                return 0;
        }
 
-       cmem = zs_map_object(zram->mem_pool, handle);
+       cmem = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
        ret = lzo1x_decompress_safe(cmem, zram->table[index].size,
                                    mem, &clen);
        zs_unmap_object(zram->mem_pool, handle);
@@ -351,7 +352,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
                ret = -ENOMEM;
                goto out;
        }
-       cmem = zs_map_object(zram->mem_pool, handle);
+       cmem = zs_map_object(zram->mem_pool, handle, ZS_MM_WO);
 
        memcpy(cmem, src, clen);
 

diff --git a/drivers/staging/zsmalloc/zsmalloc-main.c b/drivers/staging/zsmalloc/zsmalloc-main.c
index abf7c13c89c87fc289229837079f2d7fe593a090..8b0bcb626a7f8e3082fa6dce0fbac81c92fde966 100644 (file)
--- a/drivers/staging/zsmalloc/zsmalloc-main.c
+++ b/drivers/staging/zsmalloc/zsmalloc-main.c
@@ -484,9 +484,6 @@ static void zs_copy_map_object(char *buf, struct page *firstpage,
        sizes[0] = PAGE_SIZE - off;
        sizes[1] = size - sizes[0];
 
-       /* disable page faults to match kmap_atomic() return conditions */
-       pagefault_disable();
-
        /* copy object to per-cpu buffer */
        addr = kmap_atomic(pages[0]);
        memcpy(buf, addr + off, sizes[0]);
@@ -517,9 +514,6 @@ static void zs_copy_unmap_object(char *buf, struct page *firstpage,
        addr = kmap_atomic(pages[1]);
        memcpy(addr, buf + sizes[0], sizes[1]);
        kunmap_atomic(addr);
-
-       /* enable page faults to match kunmap_atomic() return conditions */
-       pagefault_enable();
 }
 
 static int zs_cpu_notifier(struct notifier_block *nb, unsigned long action,
@@ -754,7 +748,8 @@ EXPORT_SYMBOL_GPL(zs_free);
  *
  * This function returns with preemption and page faults disabled.
 */
-void *zs_map_object(struct zs_pool *pool, unsigned long handle)
+void *zs_map_object(struct zs_pool *pool, unsigned long handle,
+                       enum zs_mapmode mm)
 {
        struct page *page;
        unsigned long obj_idx, off;
@@ -778,7 +773,11 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle)
                return area->vm_addr + off;
        }
 
-       zs_copy_map_object(area->vm_buf, page, off, class->size);
+       /* disable page faults to match kmap_atomic() return conditions */
+       pagefault_disable();
+
+       if (mm != ZS_MM_WO)
+               zs_copy_map_object(area->vm_buf, page, off, class->size);
        area->vm_addr = NULL;
        return area->vm_buf;
 }
@@ -795,13 +794,16 @@ void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
        struct mapping_area *area;
 
        area = &__get_cpu_var(zs_map_area);
+       /* single-page object fastpath */
        if (area->vm_addr) {
-               /* single-page object fastpath */
                kunmap_atomic(area->vm_addr);
-               put_cpu_var(zs_map_area);
-               return;
+               goto out;
        }
 
+       /* no write fastpath */
+       if (area->vm_mm == ZS_MM_RO)
+               goto pfenable;
+
        BUG_ON(!handle);
 
        obj_handle_to_location(handle, &page, &obj_idx);
@@ -810,6 +812,11 @@ void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
        off = obj_idx_to_offset(page, obj_idx, class->size);
 
        zs_copy_unmap_object(area->vm_buf, page, off, class->size);
+
+pfenable:
+       /* enable page faults to match kunmap_atomic() return conditions */
+       pagefault_enable();
+out:
        put_cpu_var(zs_map_area);
 }
 EXPORT_SYMBOL_GPL(zs_unmap_object);

diff --git a/drivers/staging/zsmalloc/zsmalloc.h b/drivers/staging/zsmalloc/zsmalloc.h
index 485cbb1a5ac8a139a36d8a03ed05728b877d5a1e..de2e8bfbcc0622aedbf732974792f6da55153718 100644 (file)
--- a/drivers/staging/zsmalloc/zsmalloc.h
+++ b/drivers/staging/zsmalloc/zsmalloc.h
@@ -15,6 +15,17 @@
 
 #include <linux/types.h>
 
+/*
+ * zsmalloc mapping modes
+ *
+ * NOTE: These only make a difference when a mapped object spans pages
+*/
+enum zs_mapmode {
+       ZS_MM_RW, /* normal read-write mapping */
+       ZS_MM_RO, /* read-only (no copy-out at unmap time) */
+       ZS_MM_WO /* write-only (no copy-in at map time) */
+};
+
 struct zs_pool;
 
 struct zs_pool *zs_create_pool(const char *name, gfp_t flags);
@@ -23,7 +34,8 @@ void zs_destroy_pool(struct zs_pool *pool);
 unsigned long zs_malloc(struct zs_pool *pool, size_t size);
 void zs_free(struct zs_pool *pool, unsigned long obj);
 
-void *zs_map_object(struct zs_pool *pool, unsigned long handle);
+void *zs_map_object(struct zs_pool *pool, unsigned long handle,
+                       enum zs_mapmode mm);
 void zs_unmap_object(struct zs_pool *pool, unsigned long handle);
 
 u64 zs_get_total_size_bytes(struct zs_pool *pool);

diff --git a/drivers/staging/zsmalloc/zsmalloc_int.h b/drivers/staging/zsmalloc/zsmalloc_int.h
index f760dae54f283f2b0a66fd0188cff2efec2d22b0..5280517677332fb350610e211fe420c68156aa5a 100644 (file)
--- a/drivers/staging/zsmalloc/zsmalloc_int.h
+++ b/drivers/staging/zsmalloc/zsmalloc_int.h
@@ -112,6 +112,7 @@ static const int fullness_threshold_frac = 4;
 struct mapping_area {
        char *vm_buf; /* copy buffer for objects that span pages */
        char *vm_addr; /* address of kmap_atomic()'ed pages */
+       enum zs_mapmode vm_mm; /* mapping mode */
 };
 
 struct size_class {