From 46ff3d78b59288a89b19efa033d0fe1f64ca2e26 Mon Sep 17 00:00:00 2001 From: yangsheng Date: Tue, 10 Feb 2009 15:30:43 +0000 Subject: [PATCH] Branch b1_6 b=10762 i=adilger, green Rate limit watchdog. Author: Jim Garlick(LLNL) --- lnet/include/libcfs/libcfs.h | 1 + lnet/libcfs/debug.c | 3 +++ lnet/libcfs/linux/linux-proc.c | 14 ++++++++++++++ lnet/libcfs/watchdog.c | 43 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/lnet/include/libcfs/libcfs.h b/lnet/include/libcfs/libcfs.h index 5db6433..e54ca3e 100644 --- a/lnet/include/libcfs/libcfs.h +++ b/lnet/include/libcfs/libcfs.h @@ -110,6 +110,7 @@ extern unsigned int libcfs_stack; extern unsigned int libcfs_debug; extern unsigned int libcfs_printk; extern unsigned int libcfs_console_ratelimit; +extern unsigned int libcfs_watchdog_ratelimit; extern cfs_duration_t libcfs_console_max_delay; extern cfs_duration_t libcfs_console_min_delay; extern unsigned int libcfs_console_backoff; diff --git a/lnet/libcfs/debug.c b/lnet/libcfs/debug.c index 1f70749..b989216 100644 --- a/lnet/libcfs/debug.c +++ b/lnet/libcfs/debug.c @@ -105,6 +105,9 @@ EXPORT_SYMBOL(portal_enter_debugger); unsigned int libcfs_catastrophe; EXPORT_SYMBOL(libcfs_catastrophe); +unsigned int libcfs_watchdog_ratelimit = 300; +EXPORT_SYMBOL(libcfs_watchdog_ratelimit); + unsigned int libcfs_panic_on_lbug = 0; CFS_MODULE_PARM(libcfs_panic_on_lbug, "i", uint, 0644, "Lustre kernel panic on LBUG"); diff --git a/lnet/libcfs/linux/linux-proc.c b/lnet/libcfs/linux/linux-proc.c index 2cecd6e..a10a33c 100644 --- a/lnet/libcfs/linux/linux-proc.c +++ b/lnet/libcfs/linux/linux-proc.c @@ -103,6 +103,7 @@ enum { PSDEV_LNET_DAEMON_FILE, /* spool kernel debug buffer to file */ PSDEV_LNET_DEBUG_MB, /* size of debug buffer */ PSDEV_LNET_DEBUG_LOG_UPCALL, /* debug log upcall script */ + PSDEV_LNET_WATCHDOG_RATELIMIT, /* ratelimit watchdog messages */ }; #else #define CTL_LNET CTL_UNNUMBERED @@ -188,6 +189,9 @@ static int __proc_dobitmasks(void *data, int write, DECLARE_PROC_HANDLER(proc_dobitmasks) +static int min_watchdog_ratelimit = 0; /* disable ratelimiting */ +static int max_watchdog_ratelimit = (24*60*60); /* limit to once per day */ + static int __proc_dump_kernel(void *data, int write, loff_t pos, void *buffer, int nob) { @@ -454,6 +458,16 @@ static cfs_sysctl_table_t lnet_table[] = { .mode = 0644, .proc_handler = &proc_debug_mb, }, + { + .ctl_name = PSDEV_LNET_WATCHDOG_RATELIMIT, + .procname = "watchdog_ratelimit", + .data = &libcfs_watchdog_ratelimit, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_dointvec_minmax, + .extra1 = &min_watchdog_ratelimit, + .extra2 = &max_watchdog_ratelimit, + }, {0} }; diff --git a/lnet/libcfs/watchdog.c b/lnet/libcfs/watchdog.c index 78c1f9e..b007c28 100644 --- a/lnet/libcfs/watchdog.c +++ b/lnet/libcfs/watchdog.c @@ -97,6 +97,11 @@ static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */ static struct list_head lcw_pending_timers = \ LIST_HEAD_INIT(lcw_pending_timers); +/* Last time a watchdog expired */ +static cfs_time_t lcw_last_watchdog_time; +static int lcw_recent_watchdog_count; +static spinlock_t lcw_last_watchdog_lock = SPIN_LOCK_UNLOCKED; + #ifdef HAVE_TASKLIST_LOCK static void lcw_dump(struct lc_watchdog *lcw) @@ -131,6 +136,8 @@ lcw_dump(struct lc_watchdog *lcw) static void lcw_cb(unsigned long data) { struct lc_watchdog *lcw = (struct lc_watchdog *)data; + cfs_time_t current_time; + cfs_duration_t delta_time; ENTRY; @@ -140,14 +147,38 @@ static void lcw_cb(unsigned long data) } lcw->lcw_state = LC_WATCHDOG_EXPIRED; + current_time = cfs_time_current(); + + /* Check to see if we should throttle the watchdog timer to avoid + * too many dumps going to the console thus triggering an NMI. + * Normally we would not hold the spin lock over the CWARN but in + * this case we hold it to ensure non ratelimited lcw_dumps are not + * interleaved on the console making them hard to read. */ + spin_lock_bh(&lcw_last_watchdog_lock); + delta_time = cfs_duration_sec(current_time - lcw_last_watchdog_time); + + if (delta_time < libcfs_watchdog_ratelimit && lcw_recent_watchdog_count > 3) { + CWARN("Refusing to fire watchdog for pid %d: it was inactive " + "for %ldms. Rate limiting 1 per %d seconds.\n", + (int)lcw->lcw_pid,cfs_duration_sec(lcw->lcw_time) * 1000, + libcfs_watchdog_ratelimit); + } else { + if (delta_time < libcfs_watchdog_ratelimit) { + lcw_recent_watchdog_count++; + } else { + memcpy(&lcw_last_watchdog_time, ¤t_time, + sizeof(current_time)); + lcw_recent_watchdog_count = 0; + } - /* NB this warning should appear on the console, but may not get into - * the logs since we're running in a softirq handler */ - - CWARN("Watchdog triggered for pid %d: it was inactive for %lds\n", - (int)lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time)); - lcw_dump(lcw); + /* This warning should appear on the console, but may not get + * into the logs since we're running in a softirq handler */ + CWARN("Watchdog triggered for pid %d: it was inactive for %lds\n", + (int)lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time)); + lcw_dump(lcw); + } + spin_unlock_bh(&lcw_last_watchdog_lock); spin_lock_bh(&lcw_pending_timers_lock); if (list_empty(&lcw->lcw_list)) { -- 1.8.3.1