X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=libcfs%2Flibcfs%2Fwatchdog.c;h=b9f488ff59102a46bffd55d71e8baffbcc1866da;hb=fd79a11a9f4c9526af33a68c77a28437b3c29dfa;hp=6c90188ca4ec6c5c614ea9f35ff8a941c613a76a;hpb=c8616363809a73b3d4b6c736e6b55a84d7f137b4;p=fs%2Flustre-release.git diff --git a/libcfs/libcfs/watchdog.c b/libcfs/libcfs/watchdog.c index 6c90188..b9f488f 100644 --- a/libcfs/libcfs/watchdog.c +++ b/libcfs/libcfs/watchdog.c @@ -26,7 +26,7 @@ * GPL HEADER END */ /* - * Copyright 2008 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. */ /* @@ -44,16 +44,16 @@ #include "tracefile.h" struct lc_watchdog { - cfs_timer_t lcw_timer; /* kernel timer */ - struct list_head lcw_list; - struct timeval lcw_last_touched; - cfs_task_t *lcw_task; + cfs_spinlock_t lcw_lock; /* check or change lcw_list */ + int lcw_refcount; /* must hold lcw_pending_timers_lock */ + cfs_timer_t lcw_timer; /* kernel timer */ + cfs_list_t lcw_list; /* chain on pending list */ + cfs_time_t lcw_last_touched; /* last touched stamp */ + cfs_task_t *lcw_task; /* owner task */ + void (*lcw_callback)(pid_t, void *); + void *lcw_data; - void (*lcw_callback)(pid_t, void *); - void *lcw_data; - - pid_t lcw_pid; - cfs_duration_t lcw_time; /* time until watchdog fires, jiffies */ + pid_t lcw_pid; enum { LC_WATCHDOG_DISABLED, @@ -68,8 +68,8 @@ struct lc_watchdog { * and lcw_stop_completion when it exits. * Wake lcw_event_waitq to signal timer callback dispatches. */ -static struct completion lcw_start_completion; -static struct completion lcw_stop_completion; +static cfs_completion_t lcw_start_completion; +static cfs_completion_t lcw_stop_completion; static cfs_waitq_t lcw_event_waitq; /* @@ -83,54 +83,55 @@ static unsigned long lcw_flags = 0; /* * Number of outstanding watchdogs. * When it hits 1, we start the dispatcher. - * When it hits 0, we stop the distpatcher. + * When it hits 0, we stop the dispatcher. */ static __u32 lcw_refcount = 0; -static DECLARE_MUTEX(lcw_refcount_sem); +static CFS_DECLARE_MUTEX(lcw_refcount_sem); /* * List of timers that have fired that need their callbacks run by the * dispatcher. */ -static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */ -static struct list_head lcw_pending_timers = \ +/* BH lock! */ +static cfs_spinlock_t lcw_pending_timers_lock = CFS_SPIN_LOCK_UNLOCKED; +static cfs_list_t lcw_pending_timers = \ CFS_LIST_HEAD_INIT(lcw_pending_timers); -#ifdef HAVE_TASKLIST_LOCK +/* Last time a watchdog expired */ +static cfs_time_t lcw_last_watchdog_time; +static int lcw_recent_watchdog_count; + static void lcw_dump(struct lc_watchdog *lcw) { - cfs_task_t *tsk; ENTRY; - - read_lock(&tasklist_lock); - tsk = find_task_by_pid(lcw->lcw_pid); - - if (tsk == NULL) { - CWARN("Process " LPPID " was not found in the task list; " - "watchdog callback may be incomplete\n", lcw->lcw_pid); - } else if (tsk != lcw->lcw_task) { - CWARN("The current process " LPPID " did not set the watchdog; " - "watchdog callback may be incomplete\n", lcw->lcw_pid); +#if defined(HAVE_TASKLIST_LOCK) + cfs_read_lock(&tasklist_lock); +#elif defined(HAVE_TASK_RCU) + rcu_read_lock(); +#else + CERROR("unable to dump stack because of missing export\n"); + RETURN_EXIT; +#endif + if (lcw->lcw_task == NULL) { + LCONSOLE_WARN("Process " LPPID " was not found in the task " + "list; watchdog callback may be incomplete\n", + (int)lcw->lcw_pid); } else { - libcfs_debug_dumpstack(tsk); + libcfs_debug_dumpstack(lcw->lcw_task); } - - read_unlock(&tasklist_lock); + +#if defined(HAVE_TASKLIST_LOCK) + cfs_read_unlock(&tasklist_lock); +#elif defined(HAVE_TASK_RCU) + rcu_read_unlock(); +#endif EXIT; } -#else -static void -lcw_dump(struct lc_watchdog *lcw) -{ - CERROR("unable to dump stack because of missing export\n"); -} -#endif static void lcw_cb(ulong_ptr_t data) { struct lc_watchdog *lcw = (struct lc_watchdog *)data; - ENTRY; if (lcw->lcw_state != LC_WATCHDOG_ENABLED) { @@ -140,22 +141,16 @@ static void lcw_cb(ulong_ptr_t data) lcw->lcw_state = LC_WATCHDOG_EXPIRED; - /* NB this warning should appear on the console, but may not get into - * the logs since we're running in a softirq handler */ + cfs_spin_lock_bh(&lcw->lcw_lock); + LASSERT(cfs_list_empty(&lcw->lcw_list)); - CWARN("Watchdog triggered for pid: " LPPID " it was inactive for %lds\n", - lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time)); - lcw_dump(lcw); - - spin_lock_bh(&lcw_pending_timers_lock); - - if (list_empty(&lcw->lcw_list)) { - list_add(&lcw->lcw_list, &lcw_pending_timers); - cfs_waitq_signal(&lcw_event_waitq); - } - - spin_unlock_bh(&lcw_pending_timers_lock); + cfs_spin_lock_bh(&lcw_pending_timers_lock); + lcw->lcw_refcount++; /* +1 for pending list */ + cfs_list_add(&lcw->lcw_list, &lcw_pending_timers); + cfs_waitq_signal(&lcw_event_waitq); + cfs_spin_unlock_bh(&lcw_pending_timers_lock); + cfs_spin_unlock_bh(&lcw->lcw_lock); EXIT; } @@ -163,20 +158,68 @@ static int is_watchdog_fired(void) { int rc; - if (test_bit(LCW_FLAG_STOP, &lcw_flags)) + if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags)) return 1; - spin_lock_bh(&lcw_pending_timers_lock); - rc = !list_empty(&lcw_pending_timers); - spin_unlock_bh(&lcw_pending_timers_lock); + cfs_spin_lock_bh(&lcw_pending_timers_lock); + rc = !cfs_list_empty(&lcw_pending_timers); + cfs_spin_unlock_bh(&lcw_pending_timers_lock); return rc; } +static void lcw_dump_stack(struct lc_watchdog *lcw) +{ + cfs_time_t current_time; + cfs_duration_t delta_time; + struct timeval timediff; + + current_time = cfs_time_current(); + delta_time = cfs_time_sub(current_time, lcw->lcw_last_touched); + cfs_duration_usec(delta_time, &timediff); + + /* + * Check to see if we should throttle the watchdog timer to avoid + * too many dumps going to the console thus triggering an NMI. + */ + delta_time = cfs_duration_sec(cfs_time_sub(current_time, + lcw_last_watchdog_time)); + + if (delta_time < libcfs_watchdog_ratelimit && + lcw_recent_watchdog_count > 3) { + LCONSOLE_WARN("Service thread pid %u was inactive for " + "%lu.%.02lus. Watchdog stack traces are limited " + "to 3 per %d seconds, skipping this one.\n", + (int)lcw->lcw_pid, + timediff.tv_sec, + timediff.tv_usec / 10000, + libcfs_watchdog_ratelimit); + } else { + if (delta_time < libcfs_watchdog_ratelimit) { + lcw_recent_watchdog_count++; + } else { + memcpy(&lcw_last_watchdog_time, ¤t_time, + sizeof(current_time)); + lcw_recent_watchdog_count = 0; + } + + LCONSOLE_WARN("Service thread pid %u was inactive for " + "%lu.%.02lus. The thread might be hung, or it " + "might only be slow and will resume later. " + "Dumping the stack trace for debugging purposes:" + "\n", + (int)lcw->lcw_pid, + timediff.tv_sec, + timediff.tv_usec / 10000); + lcw_dump(lcw); + } +} + static int lcw_dispatch_main(void *data) { int rc = 0; unsigned long flags; struct lc_watchdog *lcw; + CFS_LIST_HEAD (zombies); ENTRY; @@ -187,17 +230,20 @@ static int lcw_dispatch_main(void *data) RECALC_SIGPENDING; SIGNAL_MASK_UNLOCK(current, flags); - complete(&lcw_start_completion); + cfs_complete(&lcw_start_completion); while (1) { - cfs_wait_event_interruptible(lcw_event_waitq, is_watchdog_fired(), rc); + int dumplog = 1; + + cfs_wait_event_interruptible(lcw_event_waitq, + is_watchdog_fired(), rc); CDEBUG(D_INFO, "Watchdog got woken up...\n"); - if (test_bit(LCW_FLAG_STOP, &lcw_flags)) { + if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags)) { CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n"); - spin_lock_bh(&lcw_pending_timers_lock); - rc = !list_empty(&lcw_pending_timers); - spin_unlock_bh(&lcw_pending_timers_lock); + cfs_spin_lock_bh(&lcw_pending_timers_lock); + rc = !cfs_list_empty(&lcw_pending_timers); + cfs_spin_unlock_bh(&lcw_pending_timers_lock); if (rc) { CERROR("pending timers list was not empty at " "time of watchdog dispatch shutdown\n"); @@ -205,27 +251,65 @@ static int lcw_dispatch_main(void *data) break; } - spin_lock_bh(&lcw_pending_timers_lock); - while (!list_empty(&lcw_pending_timers)) { + cfs_spin_lock_bh(&lcw_pending_timers_lock); + while (!cfs_list_empty(&lcw_pending_timers)) { + int is_dumplog; + + lcw = cfs_list_entry(lcw_pending_timers.next, + struct lc_watchdog, lcw_list); + /* +1 ref for callback to make sure lwc wouldn't be + * deleted after releasing lcw_pending_timers_lock */ + lcw->lcw_refcount++; + cfs_spin_unlock_bh(&lcw_pending_timers_lock); + + /* lock ordering */ + cfs_spin_lock_bh(&lcw->lcw_lock); + cfs_spin_lock_bh(&lcw_pending_timers_lock); + + if (cfs_list_empty(&lcw->lcw_list)) { + /* already removed from pending list */ + lcw->lcw_refcount--; /* -1 ref for callback */ + if (lcw->lcw_refcount == 0) + cfs_list_add(&lcw->lcw_list, &zombies); + cfs_spin_unlock_bh(&lcw->lcw_lock); + /* still hold lcw_pending_timers_lock */ + continue; + } + + cfs_list_del_init(&lcw->lcw_list); + lcw->lcw_refcount--; /* -1 ref for pending list */ - lcw = list_entry(lcw_pending_timers.next, - struct lc_watchdog, - lcw_list); - list_del_init(&lcw->lcw_list); - spin_unlock_bh(&lcw_pending_timers_lock); + cfs_spin_unlock_bh(&lcw_pending_timers_lock); + cfs_spin_unlock_bh(&lcw->lcw_lock); - CDEBUG(D_INFO, "found lcw for pid " LPPID ": inactive for " - "%lds\n", lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time)); + CDEBUG(D_INFO, "found lcw for pid " LPPID "\n", + lcw->lcw_pid); + lcw_dump_stack(lcw); - if (lcw->lcw_state != LC_WATCHDOG_DISABLED) + is_dumplog = lcw->lcw_callback == lc_watchdog_dumplog; + if (lcw->lcw_state != LC_WATCHDOG_DISABLED && + (dumplog || !is_dumplog)) { lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data); + if (dumplog && is_dumplog) + dumplog = 0; + } - spin_lock_bh(&lcw_pending_timers_lock); + cfs_spin_lock_bh(&lcw_pending_timers_lock); + lcw->lcw_refcount--; /* -1 ref for callback */ + if (lcw->lcw_refcount == 0) + cfs_list_add(&lcw->lcw_list, &zombies); + } + cfs_spin_unlock_bh(&lcw_pending_timers_lock); + + while (!cfs_list_empty(&zombies)) { + lcw = cfs_list_entry(lcw_pending_timers.next, + struct lc_watchdog, lcw_list); + cfs_list_del(&lcw->lcw_list); + LIBCFS_FREE(lcw, sizeof(*lcw)); } - spin_unlock_bh(&lcw_pending_timers_lock); } - complete(&lcw_stop_completion); + cfs_complete(&lcw_stop_completion); RETURN(rc); } @@ -237,18 +321,18 @@ static void lcw_dispatch_start(void) ENTRY; LASSERT(lcw_refcount == 1); - init_completion(&lcw_stop_completion); - init_completion(&lcw_start_completion); + cfs_init_completion(&lcw_stop_completion); + cfs_init_completion(&lcw_start_completion); cfs_waitq_init(&lcw_event_waitq); CDEBUG(D_INFO, "starting dispatch thread\n"); - rc = kernel_thread(lcw_dispatch_main, NULL, 0); + rc = cfs_create_thread(lcw_dispatch_main, NULL, 0); if (rc < 0) { CERROR("error spawning watchdog dispatch thread: %d\n", rc); EXIT; return; } - wait_for_completion(&lcw_start_completion); + cfs_wait_for_completion(&lcw_start_completion); CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n"); EXIT; @@ -261,17 +345,17 @@ static void lcw_dispatch_stop(void) CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n"); - set_bit(LCW_FLAG_STOP, &lcw_flags); + cfs_set_bit(LCW_FLAG_STOP, &lcw_flags); cfs_waitq_signal(&lcw_event_waitq); - wait_for_completion(&lcw_stop_completion); + cfs_wait_for_completion(&lcw_stop_completion); CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n"); EXIT; } -struct lc_watchdog *lc_watchdog_add(int timeout_ms, +struct lc_watchdog *lc_watchdog_add(int timeout, void (*callback)(pid_t, void *), void *data) { @@ -284,9 +368,10 @@ struct lc_watchdog *lc_watchdog_add(int timeout_ms, RETURN(ERR_PTR(-ENOMEM)); } + cfs_spin_lock_init(&lcw->lcw_lock); + lcw->lcw_refcount = 1; /* refcount for owner */ lcw->lcw_task = cfs_current(); lcw->lcw_pid = cfs_curproc_pid(); - lcw->lcw_time = cfs_time_seconds(timeout_ms) / 1000; lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog; lcw->lcw_data = data; lcw->lcw_state = LC_WATCHDOG_DISABLED; @@ -294,15 +379,15 @@ struct lc_watchdog *lc_watchdog_add(int timeout_ms, CFS_INIT_LIST_HEAD(&lcw->lcw_list); cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw); - down(&lcw_refcount_sem); + cfs_down(&lcw_refcount_sem); if (++lcw_refcount == 1) lcw_dispatch_start(); - up(&lcw_refcount_sem); + cfs_up(&lcw_refcount_sem); /* Keep this working in case we enable them by default */ if (lcw->lcw_state == LC_WATCHDOG_ENABLED) { - do_gettimeofday(&lcw->lcw_last_touched); - cfs_timer_arm(&lcw->lcw_timer, lcw->lcw_time + + lcw->lcw_last_touched = cfs_time_current(); + cfs_timer_arm(&lcw->lcw_timer, cfs_time_seconds(timeout) + cfs_time_current()); } @@ -312,45 +397,54 @@ EXPORT_SYMBOL(lc_watchdog_add); static void lcw_update_time(struct lc_watchdog *lcw, const char *message) { - struct timeval newtime; - struct timeval timediff; + cfs_time_t newtime = cfs_time_current();; - do_gettimeofday(&newtime); if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) { - cfs_timeval_sub(&newtime, &lcw->lcw_last_touched, &timediff); - CWARN("Expired watchdog for pid " LPPID " %s after %lu.%.4lus\n", - lcw->lcw_pid, - message, - timediff.tv_sec, - timediff.tv_usec / 100); + struct timeval timediff; + cfs_time_t delta_time = cfs_time_sub(newtime, + lcw->lcw_last_touched); + cfs_duration_usec(delta_time, &timediff); + + LCONSOLE_WARN("Service thread pid %u %s after %lu.%.02lus. " + "This indicates the system was overloaded (too " + "many service threads, or there were not enough " + "hardware resources).\n", + lcw->lcw_pid, + message, + timediff.tv_sec, + timediff.tv_usec / 10000); } lcw->lcw_last_touched = newtime; } -void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms) +static void lc_watchdog_del_pending(struct lc_watchdog *lcw) +{ + cfs_spin_lock_bh(&lcw->lcw_lock); + if (unlikely(!cfs_list_empty(&lcw->lcw_list))) { + cfs_spin_lock_bh(&lcw_pending_timers_lock); + cfs_list_del_init(&lcw->lcw_list); + lcw->lcw_refcount--; /* -1 ref for pending list */ + cfs_spin_unlock_bh(&lcw_pending_timers_lock); + } + + cfs_spin_unlock_bh(&lcw->lcw_lock); +} + +void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout) { ENTRY; LASSERT(lcw != NULL); - spin_lock_bh(&lcw_pending_timers_lock); - list_del_init(&lcw->lcw_list); - spin_unlock_bh(&lcw_pending_timers_lock); + lc_watchdog_del_pending(lcw); - lcw_update_time(lcw, "touched"); + lcw_update_time(lcw, "resumed"); lcw->lcw_state = LC_WATCHDOG_ENABLED; cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() + - cfs_time_seconds(timeout_ms) / 1000); + cfs_time_seconds(timeout)); EXIT; } -EXPORT_SYMBOL(lc_watchdog_touch_ms); - -/* deprecated - use above instead */ -void lc_watchdog_touch(struct lc_watchdog *lcw) -{ - lc_watchdog_touch_ms(lcw, (int)cfs_duration_sec(lcw->lcw_time) * 1000); -} EXPORT_SYMBOL(lc_watchdog_touch); void lc_watchdog_disable(struct lc_watchdog *lcw) @@ -358,12 +452,9 @@ void lc_watchdog_disable(struct lc_watchdog *lcw) ENTRY; LASSERT(lcw != NULL); - spin_lock_bh(&lcw_pending_timers_lock); - if (!list_empty(&lcw->lcw_list)) - list_del_init(&lcw->lcw_list); - spin_unlock_bh(&lcw_pending_timers_lock); + lc_watchdog_del_pending(lcw); - lcw_update_time(lcw, "disabled"); + lcw_update_time(lcw, "completed"); lcw->lcw_state = LC_WATCHDOG_DISABLED; EXIT; @@ -372,24 +463,34 @@ EXPORT_SYMBOL(lc_watchdog_disable); void lc_watchdog_delete(struct lc_watchdog *lcw) { + int dead; + ENTRY; LASSERT(lcw != NULL); cfs_timer_disarm(&lcw->lcw_timer); - lcw_update_time(lcw, "deleted"); + lcw_update_time(lcw, "stopped"); + + cfs_spin_lock_bh(&lcw->lcw_lock); + cfs_spin_lock_bh(&lcw_pending_timers_lock); + if (unlikely(!cfs_list_empty(&lcw->lcw_list))) { + cfs_list_del_init(&lcw->lcw_list); + lcw->lcw_refcount--; /* -1 ref for pending list */ + } + + lcw->lcw_refcount--; /* -1 ref for owner */ + dead = lcw->lcw_refcount == 0; + cfs_spin_unlock_bh(&lcw_pending_timers_lock); + cfs_spin_unlock_bh(&lcw->lcw_lock); - spin_lock_bh(&lcw_pending_timers_lock); - if (!list_empty(&lcw->lcw_list)) - list_del_init(&lcw->lcw_list); - spin_unlock_bh(&lcw_pending_timers_lock); + if (dead) + LIBCFS_FREE(lcw, sizeof(*lcw)); - down(&lcw_refcount_sem); + cfs_down(&lcw_refcount_sem); if (--lcw_refcount == 0) lcw_dispatch_stop(); - up(&lcw_refcount_sem); - - LIBCFS_FREE(lcw, sizeof(*lcw)); + cfs_up(&lcw_refcount_sem); EXIT; } @@ -407,7 +508,7 @@ EXPORT_SYMBOL(lc_watchdog_dumplog); #else /* !defined(WITH_WATCHDOG) */ -struct lc_watchdog *lc_watchdog_add(int timeout_ms, +struct lc_watchdog *lc_watchdog_add(int timeout, void (*callback)(pid_t pid, void *), void *data) { @@ -416,12 +517,7 @@ struct lc_watchdog *lc_watchdog_add(int timeout_ms, } EXPORT_SYMBOL(lc_watchdog_add); -void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms) -{ -} -EXPORT_SYMBOL(lc_watchdog_touch_ms); - -void lc_watchdog_touch(struct lc_watchdog *lcw) +void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout) { } EXPORT_SYMBOL(lc_watchdog_touch);