mm: make optimistic check for swapin readahead

Introduce a new sysfs integer knob
/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_swap which makes
optimistic check for swapin readahead to increase thp collapse rate.
Before getting swapped out pages to memory, checks them and allows up to a
certain number.  It also prints out using tracepoints amount of unmapped
ptes.

[vdavydov@parallels.com: fix scan not aborted on SCAN_EXCEED_SWAP_PTE]
[sfr@canb.auug.org.au: build fix]
Link: http://lkml.kernel.org/r/20160616154503.65806e12@canb.auug.org.au
Signed-off-by: Ebru Akagunduz <ebru.akagunduz@gmail.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Xie XiuQi <xiexiuqi@huawei.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: David Rientjes <rientjes@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h
index 551ba4acde4d89497aacb901b2ceba2a0fa2d6fc..fad6539c9d685b7864f40844e3e1a545a1d45a3c 100644 (file)
--- a/include/trace/events/huge_memory.h
+++ b/include/trace/events/huge_memory.h
@@ -28,7 +28,8 @@
        EM( SCAN_SWAP_CACHE_PAGE,       "page_swap_cache")              \
        EM( SCAN_DEL_PAGE_LRU,          "could_not_delete_page_from_lru")\
        EM( SCAN_ALLOC_HUGE_PAGE_FAIL,  "alloc_huge_page_failed")       \
-       EMe( SCAN_CGROUP_CHARGE_FAIL,   "ccgroup_charge_failed")
+       EM( SCAN_CGROUP_CHARGE_FAIL,    "ccgroup_charge_failed")        \
+       EMe( SCAN_EXCEED_SWAP_PTE,      "exceed_swap_pte")
 
 #undef EM
 #undef EMe
@@ -45,9 +46,9 @@ SCAN_STATUS
 TRACE_EVENT(mm_khugepaged_scan_pmd,
 
        TP_PROTO(struct mm_struct *mm, struct page *page, bool writable,
-                bool referenced, int none_or_zero, int status),
+                bool referenced, int none_or_zero, int status, int unmapped),
 
-       TP_ARGS(mm, page, writable, referenced, none_or_zero, status),
+       TP_ARGS(mm, page, writable, referenced, none_or_zero, status, unmapped),
 
        TP_STRUCT__entry(
                __field(struct mm_struct *, mm)
@@ -56,6 +57,7 @@ TRACE_EVENT(mm_khugepaged_scan_pmd,
                __field(bool, referenced)
                __field(int, none_or_zero)
                __field(int, status)
+               __field(int, unmapped)
        ),
 
        TP_fast_assign(
@@ -65,15 +67,17 @@ TRACE_EVENT(mm_khugepaged_scan_pmd,
                __entry->referenced = referenced;
                __entry->none_or_zero = none_or_zero;
                __entry->status = status;
+               __entry->unmapped = unmapped;
        ),
 
-       TP_printk("mm=%p, scan_pfn=0x%lx, writable=%d, referenced=%d, none_or_zero=%d, status=%s",
+       TP_printk("mm=%p, scan_pfn=0x%lx, writable=%d, referenced=%d, none_or_zero=%d, status=%s, unmapped=%d",
                __entry->mm,
                __entry->pfn,
                __entry->writable,
                __entry->referenced,
                __entry->none_or_zero,
-               __print_symbolic(__entry->status, SCAN_STATUS))
+               __print_symbolic(__entry->status, SCAN_STATUS),
+               __entry->unmapped)
 );
 
 TRACE_EVENT(mm_collapse_huge_page,

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 23d1bf42fef105e85e8e452d663cd6c2d5d6fdac..ed474483a620d3cea65a6322a83a80f8a72107b5 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -57,7 +57,8 @@ enum scan_result {
        SCAN_SWAP_CACHE_PAGE,
        SCAN_DEL_PAGE_LRU,
        SCAN_ALLOC_HUGE_PAGE_FAIL,
-       SCAN_CGROUP_CHARGE_FAIL
+       SCAN_CGROUP_CHARGE_FAIL,
+       SCAN_EXCEED_SWAP_PTE
 };
 
 #define CREATE_TRACE_POINTS
@@ -100,6 +101,7 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
  * fault.
  */
 static unsigned int khugepaged_max_ptes_none __read_mostly;
+static unsigned int khugepaged_max_ptes_swap __read_mostly;
 
 static int khugepaged(void *none);
 static int khugepaged_slab_init(void);
@@ -598,6 +600,33 @@ static struct kobj_attribute khugepaged_max_ptes_none_attr =
        __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
               khugepaged_max_ptes_none_store);
 
+static ssize_t khugepaged_max_ptes_swap_show(struct kobject *kobj,
+                                            struct kobj_attribute *attr,
+                                            char *buf)
+{
+       return sprintf(buf, "%u\n", khugepaged_max_ptes_swap);
+}
+
+static ssize_t khugepaged_max_ptes_swap_store(struct kobject *kobj,
+                                             struct kobj_attribute *attr,
+                                             const char *buf, size_t count)
+{
+       int err;
+       unsigned long max_ptes_swap;
+
+       err  = kstrtoul(buf, 10, &max_ptes_swap);
+       if (err || max_ptes_swap > HPAGE_PMD_NR-1)
+               return -EINVAL;
+
+       khugepaged_max_ptes_swap = max_ptes_swap;
+
+       return count;
+}
+
+static struct kobj_attribute khugepaged_max_ptes_swap_attr =
+       __ATTR(max_ptes_swap, 0644, khugepaged_max_ptes_swap_show,
+              khugepaged_max_ptes_swap_store);
+
 static struct attribute *khugepaged_attr[] = {
        &khugepaged_defrag_attr.attr,
        &khugepaged_max_ptes_none_attr.attr,
@@ -606,6 +635,7 @@ static struct attribute *khugepaged_attr[] = {
        &full_scans_attr.attr,
        &scan_sleep_millisecs_attr.attr,
        &alloc_sleep_millisecs_attr.attr,
+       &khugepaged_max_ptes_swap_attr.attr,
        NULL,
 };
 
@@ -674,6 +704,7 @@ static int __init hugepage_init(void)
 
        khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
        khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;
+       khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
        /*
         * hugepages can't be allocated by the buddy allocator
         */
@@ -2507,7 +2538,7 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
        struct page *page = NULL;
        unsigned long _address;
        spinlock_t *ptl;
-       int node = NUMA_NO_NODE;
+       int node = NUMA_NO_NODE, unmapped = 0;
        bool writable = false, referenced = false;
 
        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
@@ -2523,6 +2554,14 @@ static int khugepaged_scan_pmd(struct mm_struct *mm,
        for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
             _pte++, _address += PAGE_SIZE) {
                pte_t pteval = *_pte;
+               if (is_swap_pte(pteval)) {
+                       if (++unmapped <= khugepaged_max_ptes_swap) {
+                               continue;
+                       } else {
+                               result = SCAN_EXCEED_SWAP_PTE;
+                               goto out_unmap;
+                       }
+               }
                if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
                        if (!userfaultfd_armed(vma) &&
                            ++none_or_zero <= khugepaged_max_ptes_none) {
@@ -2609,7 +2648,7 @@ out_unmap:
        }
 out:
        trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
-                                    none_or_zero, result);
+                                    none_or_zero, result, unmapped);
        return ret;
 }