mm/kmemleak: turn kmemleak_lock and object->lock to raw_spinlock_t

kmemleak_lock as a rwlock on RT can possibly be acquired in atomic
context which does work.

Since the kmemleak operation is performed in atomic context make it a
raw_spinlock_t so it can also be acquired on RT.  This is used for
debugging and is not enabled by default in a production like environment
(where performance/latency matters) so it makes sense to make it a
raw_spinlock_t instead trying to get rid of the atomic context.  Turn
also the kmemleak_object->lock into raw_spinlock_t which is acquired
(nested) while the kmemleak_lock is held.

The time spent in "echo scan > kmemleak" slightly improved on 64core box
with this patch applied after boot.

[bigeasy@linutronix.de: redo the description, update comments. Merge the individual bits:  He Zhe did the kmemleak_lock, Liu Haitao the ->lock and Yongxin Liu forwarded Liu's patch.]
Link: http://lkml.kernel.org/r/20191219170834.4tah3prf2gdothz4@linutronix.de
Link: https://lkml.kernel.org/r/20181218150744.GB20197@arrakis.emea.arm.com
Link: https://lkml.kernel.org/r/1542877459-144382-1-git-send-email-zhe.he@windriver.com
Link: https://lkml.kernel.org/r/20190927082230.34152-1-yongxin.liu@windriver.com
Signed-off-by: He Zhe <zhe.he@windriver.com>
Signed-off-by: Liu Haitao <haitao.liu@windriver.com>
Signed-off-by: Yongxin Liu <yongxin.liu@windriver.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 244607663363141d170f50e0b736d9fdc20c712c..3a4259eeb5a0e5840fea6d16a2bae3c8ae215741 100644 (file)
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -13,7 +13,7 @@
  *
  * The following locks and mutexes are used by kmemleak:
  *
- * - kmemleak_lock (rwlock): protects the object_list modifications and
+ * - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
  *   accesses to the object_tree_root. The object_list is the main list
  *   holding the metadata (struct kmemleak_object) for the allocated memory
  *   blocks. The object_tree_root is a red black tree used to look-up
@@ -22,13 +22,13 @@
  *   object_tree_root in the create_object() function called from the
  *   kmemleak_alloc() callback and removed in delete_object() called from the
  *   kmemleak_free() callback
- * - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to
- *   the metadata (e.g. count) are protected by this lock. Note that some
- *   members of this structure may be protected by other means (atomic or
- *   kmemleak_lock). This lock is also held when scanning the corresponding
- *   memory block to avoid the kernel freeing it via the kmemleak_free()
- *   callback. This is less heavyweight than holding a global lock like
- *   kmemleak_lock during scanning
+ * - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
+ *   Accesses to the metadata (e.g. count) are protected by this lock. Note
+ *   that some members of this structure may be protected by other means
+ *   (atomic or kmemleak_lock). This lock is also held when scanning the
+ *   corresponding memory block to avoid the kernel freeing it via the
+ *   kmemleak_free() callback. This is less heavyweight than holding a global
+ *   lock like kmemleak_lock during scanning.
  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
  *   unreferenced objects at a time. The gray_list contains the objects which
  *   are already referenced or marked as false positives and need to be
@@ -135,7 +135,7 @@ struct kmemleak_scan_area {
  * (use_count) and freed using the RCU mechanism.
  */
 struct kmemleak_object {
-       spinlock_t lock;
+       raw_spinlock_t lock;
        unsigned int flags;             /* object status flags */
        struct list_head object_list;
        struct list_head gray_list;
@@ -191,8 +191,8 @@ static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
 static LIST_HEAD(mem_pool_free_list);
 /* search tree for object boundaries */
 static struct rb_root object_tree_root = RB_ROOT;
-/* rw_lock protecting the access to object_list and object_tree_root */
-static DEFINE_RWLOCK(kmemleak_lock);
+/* protecting the access to object_list and object_tree_root */
+static DEFINE_RAW_SPINLOCK(kmemleak_lock);
 
 /* allocation caches for kmemleak internal data */
 static struct kmem_cache *object_cache;
@@ -426,7 +426,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
        }
 
        /* slab allocation failed, try the memory pool */
-       write_lock_irqsave(&kmemleak_lock, flags);
+       raw_spin_lock_irqsave(&kmemleak_lock, flags);
        object = list_first_entry_or_null(&mem_pool_free_list,
                                          typeof(*object), object_list);
        if (object)
@@ -435,7 +435,7 @@ static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
                object = &mem_pool[--mem_pool_free_count];
        else
                pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
-       write_unlock_irqrestore(&kmemleak_lock, flags);
+       raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
 
        return object;
 }
@@ -453,9 +453,9 @@ static void mem_pool_free(struct kmemleak_object *object)
        }
 
        /* add the object to the memory pool free list */
-       write_lock_irqsave(&kmemleak_lock, flags);
+       raw_spin_lock_irqsave(&kmemleak_lock, flags);
        list_add(&object->object_list, &mem_pool_free_list);
-       write_unlock_irqrestore(&kmemleak_lock, flags);
+       raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
 }
 
 /*
@@ -514,9 +514,9 @@ static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
        struct kmemleak_object *object;
 
        rcu_read_lock();
-       read_lock_irqsave(&kmemleak_lock, flags);
+       raw_spin_lock_irqsave(&kmemleak_lock, flags);
        object = lookup_object(ptr, alias);
-       read_unlock_irqrestore(&kmemleak_lock, flags);
+       raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
 
        /* check whether the object is still available */
        if (object && !get_object(object))
@@ -546,11 +546,11 @@ static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int ali
        unsigned long flags;
        struct kmemleak_object *object;
 
-       write_lock_irqsave(&kmemleak_lock, flags);
+       raw_spin_lock_irqsave(&kmemleak_lock, flags);
        object = lookup_object(ptr, alias);
        if (object)
                __remove_object(object);
-       write_unlock_irqrestore(&kmemleak_lock, flags);
+       raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
 
        return object;
 }
@@ -585,7 +585,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
        INIT_LIST_HEAD(&object->object_list);
        INIT_LIST_HEAD(&object->gray_list);
        INIT_HLIST_HEAD(&object->area_list);
-       spin_lock_init(&object->lock);
+       raw_spin_lock_init(&object->lock);
        atomic_set(&object->use_count, 1);
        object->flags = OBJECT_ALLOCATED;
        object->pointer = ptr;
@@ -617,7 +617,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
        /* kernel backtrace */
        object->trace_len = __save_stack_trace(object->trace);
 
-       write_lock_irqsave(&kmemleak_lock, flags);
+       raw_spin_lock_irqsave(&kmemleak_lock, flags);
 
        untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
        min_addr = min(min_addr, untagged_ptr);
@@ -649,7 +649,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
 
        list_add_tail_rcu(&object->object_list, &object_list);
 out:
-       write_unlock_irqrestore(&kmemleak_lock, flags);
+       raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
        return object;
 }
 
@@ -667,9 +667,9 @@ static void __delete_object(struct kmemleak_object *object)
         * Locking here also ensures that the corresponding memory block
         * cannot be freed when it is being scanned.
         */
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        object->flags &= ~OBJECT_ALLOCATED;
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
        put_object(object);
 }
 
@@ -739,9 +739,9 @@ static void paint_it(struct kmemleak_object *object, int color)
 {
        unsigned long flags;
 
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        __paint_it(object, color);
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
 }
 
 static void paint_ptr(unsigned long ptr, int color)
@@ -798,7 +798,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
        if (scan_area_cache)
                area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
 
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        if (!area) {
                pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
                /* mark the object for full scan to avoid false positives */
@@ -820,7 +820,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
 
        hlist_add_head(&area->node, &object->area_list);
 out_unlock:
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
        put_object(object);
 }
 
@@ -842,9 +842,9 @@ static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
                return;
        }
 
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        object->excess_ref = excess_ref;
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
        put_object(object);
 }
 
@@ -864,9 +864,9 @@ static void object_no_scan(unsigned long ptr)
                return;
        }
 
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        object->flags |= OBJECT_NO_SCAN;
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
        put_object(object);
 }
 
@@ -1026,9 +1026,9 @@ void __ref kmemleak_update_trace(const void *ptr)
                return;
        }
 
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        object->trace_len = __save_stack_trace(object->trace);
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
 
        put_object(object);
 }
@@ -1233,7 +1233,7 @@ static void scan_block(void *_start, void *_end,
        unsigned long flags;
        unsigned long untagged_ptr;
 
-       read_lock_irqsave(&kmemleak_lock, flags);
+       raw_spin_lock_irqsave(&kmemleak_lock, flags);
        for (ptr = start; ptr < end; ptr++) {
                struct kmemleak_object *object;
                unsigned long pointer;
@@ -1268,7 +1268,7 @@ static void scan_block(void *_start, void *_end,
                 * previously acquired in scan_object(). These locks are
                 * enclosed by scan_mutex.
                 */
-               spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
+               raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
                /* only pass surplus references (object already gray) */
                if (color_gray(object)) {
                        excess_ref = object->excess_ref;
@@ -1277,7 +1277,7 @@ static void scan_block(void *_start, void *_end,
                        excess_ref = 0;
                        update_refs(object);
                }
-               spin_unlock(&object->lock);
+               raw_spin_unlock(&object->lock);
 
                if (excess_ref) {
                        object = lookup_object(excess_ref, 0);
@@ -1286,12 +1286,12 @@ static void scan_block(void *_start, void *_end,
                        if (object == scanned)
                                /* circular reference, ignore */
                                continue;
-                       spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
+                       raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
                        update_refs(object);
-                       spin_unlock(&object->lock);
+                       raw_spin_unlock(&object->lock);
                }
        }
-       read_unlock_irqrestore(&kmemleak_lock, flags);
+       raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
 }
 
 /*
@@ -1324,7 +1324,7 @@ static void scan_object(struct kmemleak_object *object)
         * Once the object->lock is acquired, the corresponding memory block
         * cannot be freed (the same lock is acquired in delete_object).
         */
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        if (object->flags & OBJECT_NO_SCAN)
                goto out;
        if (!(object->flags & OBJECT_ALLOCATED))
@@ -1344,9 +1344,9 @@ static void scan_object(struct kmemleak_object *object)
                        if (start >= end)
                                break;
 
-                       spin_unlock_irqrestore(&object->lock, flags);
+                       raw_spin_unlock_irqrestore(&object->lock, flags);
                        cond_resched();
-                       spin_lock_irqsave(&object->lock, flags);
+                       raw_spin_lock_irqsave(&object->lock, flags);
                } while (object->flags & OBJECT_ALLOCATED);
        } else
                hlist_for_each_entry(area, &object->area_list, node)
@@ -1354,7 +1354,7 @@ static void scan_object(struct kmemleak_object *object)
                                   (void *)(area->start + area->size),
                                   object);
 out:
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
 }
 
 /*
@@ -1407,7 +1407,7 @@ static void kmemleak_scan(void)
        /* prepare the kmemleak_object's */
        rcu_read_lock();
        list_for_each_entry_rcu(object, &object_list, object_list) {
-               spin_lock_irqsave(&object->lock, flags);
+               raw_spin_lock_irqsave(&object->lock, flags);
 #ifdef DEBUG
                /*
                 * With a few exceptions there should be a maximum of
@@ -1424,7 +1424,7 @@ static void kmemleak_scan(void)
                if (color_gray(object) && get_object(object))
                        list_add_tail(&object->gray_list, &gray_list);
 
-               spin_unlock_irqrestore(&object->lock, flags);
+               raw_spin_unlock_irqrestore(&object->lock, flags);
        }
        rcu_read_unlock();
 
@@ -1492,14 +1492,14 @@ static void kmemleak_scan(void)
         */
        rcu_read_lock();
        list_for_each_entry_rcu(object, &object_list, object_list) {
-               spin_lock_irqsave(&object->lock, flags);
+               raw_spin_lock_irqsave(&object->lock, flags);
                if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
                    && update_checksum(object) && get_object(object)) {
                        /* color it gray temporarily */
                        object->count = object->min_count;
                        list_add_tail(&object->gray_list, &gray_list);
                }
-               spin_unlock_irqrestore(&object->lock, flags);
+               raw_spin_unlock_irqrestore(&object->lock, flags);
        }
        rcu_read_unlock();
 
@@ -1519,7 +1519,7 @@ static void kmemleak_scan(void)
         */
        rcu_read_lock();
        list_for_each_entry_rcu(object, &object_list, object_list) {
-               spin_lock_irqsave(&object->lock, flags);
+               raw_spin_lock_irqsave(&object->lock, flags);
                if (unreferenced_object(object) &&
                    !(object->flags & OBJECT_REPORTED)) {
                        object->flags |= OBJECT_REPORTED;
@@ -1529,7 +1529,7 @@ static void kmemleak_scan(void)
 
                        new_leaks++;
                }
-               spin_unlock_irqrestore(&object->lock, flags);
+               raw_spin_unlock_irqrestore(&object->lock, flags);
        }
        rcu_read_unlock();
 
@@ -1681,10 +1681,10 @@ static int kmemleak_seq_show(struct seq_file *seq, void *v)
        struct kmemleak_object *object = v;
        unsigned long flags;
 
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
                print_unreferenced(seq, object);
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
        return 0;
 }
 
@@ -1714,9 +1714,9 @@ static int dump_str_object_info(const char *str)
                return -EINVAL;
        }
 
-       spin_lock_irqsave(&object->lock, flags);
+       raw_spin_lock_irqsave(&object->lock, flags);
        dump_object_info(object);
-       spin_unlock_irqrestore(&object->lock, flags);
+       raw_spin_unlock_irqrestore(&object->lock, flags);
 
        put_object(object);
        return 0;
@@ -1735,11 +1735,11 @@ static void kmemleak_clear(void)
 
        rcu_read_lock();
        list_for_each_entry_rcu(object, &object_list, object_list) {
-               spin_lock_irqsave(&object->lock, flags);
+               raw_spin_lock_irqsave(&object->lock, flags);
                if ((object->flags & OBJECT_REPORTED) &&
                    unreferenced_object(object))
                        __paint_it(object, KMEMLEAK_GREY);
-               spin_unlock_irqrestore(&object->lock, flags);
+               raw_spin_unlock_irqrestore(&object->lock, flags);
        }
        rcu_read_unlock();