kernel/hung_task.c: allow to set checking interval separately from timeout

Currently task hung checking interval is equal to timeout, as the result
hung is detected anywhere between timeout and 2*timeout.  This is fine for
most interactive environments, but this hurts automated testing setups
(syzbot).  In an automated setup we need to strictly order CPU lockup <
RCU stall < workqueue lockup < task hung < silent loss, so that RCU stall
is not detected as task hung and task hung is not detected as silent
machine loss.  The large variance in task hung detection timeout requires
setting silent machine loss timeout to a very large value (e.g.  if task
hung is 3 mins, then silent loss need to be set to ~7 mins).  The
additional 3 minutes significantly reduce testing efficiency because
usually we crash kernel within a minute, and this can add hours to bug
localization process as it needs to do dozens of tests.

Allow setting checking interval separately from timeout.  This allows to
set timeout to, say, 3 minutes, but checking interval to 10 secs.

The interval is controlled via a new hung_task_check_interval_secs sysctl,
similar to the existing hung_task_timeout_secs sysctl.  The default value
of 0 results in the current behavior: checking interval is equal to
timeout.

[akpm@linux-foundation.org: update hung_task_timeout_max's comment]
Link: http://lkml.kernel.org/r/20180611111004.203513-1-dvyukov@google.com
Signed-off-by: Dmitry Vyukov <dvyukov@google.com>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt
index 59585030cbafa4e6a489980caf54234b5c91eeef..9d38e75b19a0790f94e724ed228d19d36dba856d 100644 (file)
--- a/Documentation/sysctl/kernel.txt
+++ b/Documentation/sysctl/kernel.txt
@@ -38,6 +38,7 @@ show up in /proc/sys/kernel:
 - hung_task_panic
 - hung_task_check_count
 - hung_task_timeout_secs
+- hung_task_check_interval_secs
 - hung_task_warnings
 - hyperv_record_panic_msg
 - kexec_load_disabled
@@ -355,7 +356,7 @@ This file shows up if CONFIG_DETECT_HUNG_TASK is enabled.
 
 hung_task_timeout_secs:
 
-Check interval. When a task in D state did not get scheduled
+When a task in D state did not get scheduled
 for more than this value report a warning.
 This file shows up if CONFIG_DETECT_HUNG_TASK is enabled.
 
@@ -364,6 +365,18 @@ Possible values to set are in range {0..LONG_MAX/HZ}.
 
 ==============================================================
 
+hung_task_check_interval_secs:
+
+Hung task check interval. If hung task checking is enabled
+(see hung_task_timeout_secs), the check is done every
+hung_task_check_interval_secs seconds.
+This file shows up if CONFIG_DETECT_HUNG_TASK is enabled.
+
+0 (default): means use hung_task_timeout_secs as checking interval.
+Possible values to set are in range {0..LONG_MAX/HZ}.
+
+==============================================================
+
 hung_task_warnings:
 
 The maximum number of warnings to report. During a check interval

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 789923fbee3a8a8dc849c92a08ce37893e175b31..58eb3a2bc6954cebec6f05113568842e32690bf7 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -853,6 +853,7 @@ struct task_struct {
 #endif
 #ifdef CONFIG_DETECT_HUNG_TASK
        unsigned long                   last_switch_count;
+       unsigned long                   last_switch_time;
 #endif
        /* Filesystem information: */
        struct fs_struct                *fs;

diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index 913488d828cb609e15d3bb4f379b49f60f62a659..a9c32daeb9d88376a4e7ce79e7de13534c95b2ea 100644 (file)
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -10,6 +10,7 @@ struct ctl_table;
 extern int          sysctl_hung_task_check_count;
 extern unsigned int  sysctl_hung_task_panic;
 extern unsigned long sysctl_hung_task_timeout_secs;
+extern unsigned long sysctl_hung_task_check_interval_secs;
 extern int sysctl_hung_task_warnings;
 extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
                                         void __user *buffer,

diff --git a/kernel/fork.c b/kernel/fork.c
index 8c760effa42ea89cbbcdb76dd9b7146462cbdcd4..8bcef2413f1ba7d1c337e8df5447257000ec77b1 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1302,6 +1302,7 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk)
        tsk->nvcsw = tsk->nivcsw = 0;
 #ifdef CONFIG_DETECT_HUNG_TASK
        tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
+       tsk->last_switch_time = 0;
 #endif
 
        tsk->mm = NULL;

diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 32b479468e4d50d69530ad766ba7e3897afa5cc3..b9132d1269ef1967950a804f7e21f3922cb491c8 100644 (file)
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -40,6 +40,11 @@ int __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT;
  */
 unsigned long __read_mostly sysctl_hung_task_timeout_secs = CONFIG_DEFAULT_HUNG_TASK_TIMEOUT;
 
+/*
+ * Zero (default value) means use sysctl_hung_task_timeout_secs:
+ */
+unsigned long __read_mostly sysctl_hung_task_check_interval_secs;
+
 int __read_mostly sysctl_hung_task_warnings = 10;
 
 static int __read_mostly did_panic;
@@ -98,8 +103,11 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout)
 
        if (switch_count != t->last_switch_count) {
                t->last_switch_count = switch_count;
+               t->last_switch_time = jiffies;
                return;
        }
+       if (time_is_after_jiffies(t->last_switch_time + timeout * HZ))
+               return;
 
        trace_sched_process_hang(t);
 
@@ -245,8 +253,13 @@ static int watchdog(void *dummy)
 
        for ( ; ; ) {
                unsigned long timeout = sysctl_hung_task_timeout_secs;
-               long t = hung_timeout_jiffies(hung_last_checked, timeout);
+               unsigned long interval = sysctl_hung_task_check_interval_secs;
+               long t;
 
+               if (interval == 0)
+                       interval = timeout;
+               interval = min_t(unsigned long, interval, timeout);
+               t = hung_timeout_jiffies(hung_last_checked, interval);
                if (t <= 0) {
                        if (!atomic_xchg(&reset_hung_task, 0))
                                check_hung_uninterruptible_tasks(timeout);

diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index f22f76b7a138ceedcc5392a24f493ec04366bc1b..dbdcb6673b27051fcdb9754eb6385cbfbf00ba5c 100644 (file)
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -145,7 +145,10 @@ static int minolduid;
 static int ngroups_max = NGROUPS_MAX;
 static const int cap_last_cap = CAP_LAST_CAP;
 
-/*this is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs */
+/*
+ * This is needed for proc_doulongvec_minmax of sysctl_hung_task_timeout_secs
+ * and hung_task_check_interval_secs
+ */
 #ifdef CONFIG_DETECT_HUNG_TASK
 static unsigned long hung_task_timeout_max = (LONG_MAX/HZ);
 #endif
@@ -1090,6 +1093,14 @@ static struct ctl_table kern_table[] = {
                .proc_handler   = proc_dohung_task_timeout_secs,
                .extra2         = &hung_task_timeout_max,
        },
+       {
+               .procname       = "hung_task_check_interval_secs",
+               .data           = &sysctl_hung_task_check_interval_secs,
+               .maxlen         = sizeof(unsigned long),
+               .mode           = 0644,
+               .proc_handler   = proc_dohung_task_timeout_secs,
+               .extra2         = &hung_task_timeout_max,
+       },
        {
                .procname       = "hung_task_warnings",
                .data           = &sysctl_hung_task_warnings,