Whamcloud - gitweb
LU-8066 libcfs: call kernel_param_unlock on error
[fs/lustre-release.git] / libcfs / libcfs / watchdog.c
index 6c1801a..359ca94 100644 (file)
@@ -1,6 +1,4 @@
-/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
- * vim:expandtab:shiftwidth=8:tabstop=8:
- *
+/*
  * GPL HEADER START
  *
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * You should have received a copy of the GNU General Public License
  * version 2 along with this program; If not, see
- * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
+ * http://www.gnu.org/licenses/gpl-2.0.html
  *
  * GPL HEADER END
  */
 /*
- * Copyright  2008 Sun Microsystems, Inc. All rights reserved
+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright (c) 2012, 2014, Intel Corporation.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
 
 #define DEBUG_SUBSYSTEM S_LNET
 
+#include <linux/kthread.h>
 #include <libcfs/libcfs.h>
 #include "tracefile.h"
 
 struct lc_watchdog {
-        cfs_timer_t       lcw_timer; /* kernel timer */
-        struct list_head  lcw_list;
-        cfs_time_t        lcw_last_touched;
-        cfs_task_t       *lcw_task;
-
-        void            (*lcw_callback)(pid_t, void *);
-        void             *lcw_data;
-
-        pid_t             lcw_pid;
-        cfs_duration_t    lcw_time; /* time until watchdog fires, jiffies */
-
-        enum {
-                LC_WATCHDOG_DISABLED,
-                LC_WATCHDOG_ENABLED,
-                LC_WATCHDOG_EXPIRED
-        } lcw_state;
+       spinlock_t              lcw_lock;       /* check or change lcw_list */
+       int                     lcw_refcount;   /* must hold lcw_pending_timers_lock */
+       struct timer_list       lcw_timer;      /* kernel timer */
+       struct list_head        lcw_list;       /* chain on pending list */
+       ktime_t                 lcw_last_touched;/* last touched stamp */
+       struct task_struct     *lcw_task;       /* owner task */
+       void                    (*lcw_callback)(pid_t, void *);
+       void                    *lcw_data;
+
+       pid_t                   lcw_pid;
+
+       enum {
+               LC_WATCHDOG_DISABLED,
+               LC_WATCHDOG_ENABLED,
+               LC_WATCHDOG_EXPIRED
+       } lcw_state;
 };
 
 #ifdef WITH_WATCHDOG
@@ -69,8 +66,8 @@ struct lc_watchdog {
  * Wake lcw_event_waitq to signal timer callback dispatches.
  */
 static struct completion lcw_start_completion;
-static struct completion lcw_stop_completion;
-static cfs_waitq_t lcw_event_waitq;
+static struct completion  lcw_stop_completion;
+static wait_queue_head_t lcw_event_waitq;
 
 /*
  * Set this and wake lcw_event_waitq to stop the dispatcher.
@@ -83,65 +80,43 @@ static unsigned long lcw_flags = 0;
 /*
  * Number of outstanding watchdogs.
  * When it hits 1, we start the dispatcher.
- * When it hits 0, we stop the distpatcher.
+ * When it hits 0, we stop the dispatcher.
  */
 static __u32         lcw_refcount = 0;
-static DECLARE_MUTEX(lcw_refcount_sem);
+static DEFINE_MUTEX(lcw_refcount_mutex);
 
 /*
  * List of timers that have fired that need their callbacks run by the
  * dispatcher.
  */
-static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */
-static struct list_head lcw_pending_timers = \
-        CFS_LIST_HEAD_INIT(lcw_pending_timers);
+/* BH lock! */
+static DEFINE_SPINLOCK(lcw_pending_timers_lock);
+static struct list_head lcw_pending_timers = LIST_HEAD_INIT(lcw_pending_timers);
 
 /* Last time a watchdog expired */
-static cfs_time_t lcw_last_watchdog_time;
+static time64_t lcw_last_watchdog_time;
 static int lcw_recent_watchdog_count;
-static spinlock_t lcw_last_watchdog_lock = SPIN_LOCK_UNLOCKED;
 
 static void
 lcw_dump(struct lc_watchdog *lcw)
 {
-        cfs_task_t *tsk;
-#if defined(HAVE_TASKLIST_LOCK)
-        read_lock(&tasklist_lock);
-#elif defined(HAVE_TASK_RCU)
-        rcu_read_lock();
-#else
-        CERROR("unable to dump stack because of missing export\n"); 
-        return;
-#endif
         ENTRY;
-
-        tsk = find_task_by_pid(lcw->lcw_pid);
-
-        if (tsk == NULL) {
-                CWARN("Process " LPPID " was not found in the task list; "
-                      "watchdog callback may be incomplete\n", lcw->lcw_pid);
-        } else if (tsk != lcw->lcw_task) {
-                CWARN("The current process " LPPID " did not set the watchdog; "
-                      "watchdog callback may be incomplete\n", lcw->lcw_pid);
+        rcu_read_lock();
+       if (lcw->lcw_task == NULL) {
+               LCONSOLE_WARN("Process %d was not found in the task "
+                              "list; watchdog callback may be incomplete\n",
+                              (int)lcw->lcw_pid);
         } else {
-                libcfs_debug_dumpstack(tsk);
+                libcfs_debug_dumpstack(lcw->lcw_task);
         }
 
-#if defined(HAVE_TASKLIST_LOCK)
-        read_unlock(&tasklist_lock);
-#elif defined(HAVE_TASK_RCU)
         rcu_read_unlock();
-#endif
         EXIT;
 }
 
-static void lcw_cb(ulong_ptr_t data)
+static void lcw_cb(uintptr_t data)
 {
         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
-        cfs_time_t current_time;
-        cfs_duration_t delta_time;
-        struct timeval timediff;
-
         ENTRY;
 
         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
@@ -150,169 +125,219 @@ static void lcw_cb(ulong_ptr_t data)
         }
 
         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
-        current_time = cfs_time_current();
-
-        delta_time = cfs_time_sub(current_time, lcw->lcw_last_touched);
-        cfs_duration_usec(delta_time, &timediff);
-
-        /* Check to see if we should throttle the watchdog timer to avoid
-         * too many dumps going to the console thus triggering an NMI.
-         * Normally we would not hold the spin lock over the CWARN but in
-         * this case we hold it to ensure non ratelimited lcw_dumps are not
-         * interleaved on the console making them hard to read. */
-        spin_lock_bh(&lcw_last_watchdog_lock);
-        delta_time = cfs_duration_sec(cfs_time_sub(current_time,
-                                                   lcw_last_watchdog_time));
-
-        if (delta_time < libcfs_watchdog_ratelimit &&
-            lcw_recent_watchdog_count > 3) {
-                CWARN("Refusing to fire watchdog for pid %d: it was inactive "
-                      "for %lu.%.02lus. Rate limiting 1 per %d seconds.\n",
-                      (int)lcw->lcw_pid, timediff.tv_sec,
-                      timediff.tv_usec / 10000, libcfs_watchdog_ratelimit);
-        } else {
-                if (delta_time < libcfs_watchdog_ratelimit) {
-                        lcw_recent_watchdog_count++;
-                } else {
-                        memcpy(&lcw_last_watchdog_time, &current_time,
-                               sizeof(current_time));
-                        lcw_recent_watchdog_count = 0;
-                }
-
-               /* This warning should appear on the console, but may not get
-                * into the logs since we're running in a softirq handler */
-                CWARN("Watchdog triggered for pid %d: it was inactive for "
-                      "%lu.%.02lus\n", (int)lcw->lcw_pid, timediff.tv_sec,
-                      timediff.tv_usec / 10000);
-                lcw_dump(lcw);
-       }
 
-        spin_unlock_bh(&lcw_last_watchdog_lock);
-        spin_lock_bh(&lcw_pending_timers_lock);
-
-        if (list_empty(&lcw->lcw_list)) {
-                list_add(&lcw->lcw_list, &lcw_pending_timers);
-                cfs_waitq_signal(&lcw_event_waitq);
-        }
+       spin_lock_bh(&lcw->lcw_lock);
+       LASSERT(list_empty(&lcw->lcw_list));
 
-        spin_unlock_bh(&lcw_pending_timers_lock);
+       spin_lock_bh(&lcw_pending_timers_lock);
+       lcw->lcw_refcount++; /* +1 for pending list */
+       list_add(&lcw->lcw_list, &lcw_pending_timers);
+       wake_up(&lcw_event_waitq);
 
-        EXIT;
+       spin_unlock_bh(&lcw_pending_timers_lock);
+       spin_unlock_bh(&lcw->lcw_lock);
+       EXIT;
 }
 
 static int is_watchdog_fired(void)
 {
-        int rc;
+       int rc;
+
+       if (test_bit(LCW_FLAG_STOP, &lcw_flags))
+               return 1;
+
+       spin_lock_bh(&lcw_pending_timers_lock);
+       rc = !list_empty(&lcw_pending_timers);
+       spin_unlock_bh(&lcw_pending_timers_lock);
+       return rc;
+}
+
+static void lcw_dump_stack(struct lc_watchdog *lcw)
+{
+       time64_t current_time = ktime_get_seconds();
+       struct timespec64 timediff;
+       time64_t delta_time;
+
+       timediff = ktime_to_timespec64(ktime_sub(ktime_get(),
+                                      lcw->lcw_last_touched));
+
+       /* LU-9235: Don't dump stack if the thread is just touched. */
+       if (timediff.tv_sec == 0)
+               return;
+
+       /*
+        * Check to see if we should throttle the watchdog timer to avoid
+        * too many dumps going to the console thus triggering an NMI.
+        */
+       delta_time = current_time - lcw_last_watchdog_time;
+       if (delta_time < libcfs_watchdog_ratelimit &&
+           lcw_recent_watchdog_count > 3) {
+               LCONSOLE_WARN("Service thread pid %u was inactive for %lu.%.02lus. Watchdog stack traces are limited to 3 per %d seconds, skipping this one.\n",
+                             (int)lcw->lcw_pid,
+                             timediff.tv_sec,
+                             timediff.tv_nsec / (NSEC_PER_SEC / 100),
+                             libcfs_watchdog_ratelimit);
+       } else {
+               if (delta_time < libcfs_watchdog_ratelimit) {
+                       lcw_recent_watchdog_count++;
+               } else {
+                       memcpy(&lcw_last_watchdog_time, &current_time,
+                              sizeof(current_time));
+                       lcw_recent_watchdog_count = 0;
+               }
+
+               LCONSOLE_WARN("Service thread pid %u was inactive for %lu.%.02lus. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:\n",
+                             (int)lcw->lcw_pid,
+                             timediff.tv_sec,
+                             timediff.tv_nsec / (NSEC_PER_SEC / 100));
+               lcw_dump(lcw);
+       }
+}
 
-        if (test_bit(LCW_FLAG_STOP, &lcw_flags))
-                return 1;
+/*
+ * Provided watchdog handlers
+ */
 
-        spin_lock_bh(&lcw_pending_timers_lock);
-        rc = !list_empty(&lcw_pending_timers);
-        spin_unlock_bh(&lcw_pending_timers_lock);
-        return rc;
+static void lc_watchdog_dumplog(pid_t pid, void *data)
+{
+       libcfs_debug_dumplog_internal((void *)((uintptr_t)pid));
 }
 
 static int lcw_dispatch_main(void *data)
 {
         int                 rc = 0;
-        unsigned long       flags;
         struct lc_watchdog *lcw;
+       struct list_head zombies = LIST_HEAD_INIT(zombies);
 
         ENTRY;
 
-        cfs_daemonize("lc_watchdogd");
-
-        SIGNAL_MASK_LOCK(current, flags);
-        sigfillset(&current->blocked);
-        RECALC_SIGPENDING;
-        SIGNAL_MASK_UNLOCK(current, flags);
-
-        complete(&lcw_start_completion);
+       complete(&lcw_start_completion);
 
         while (1) {
-                cfs_wait_event_interruptible(lcw_event_waitq, is_watchdog_fired(), rc);
-                CDEBUG(D_INFO, "Watchdog got woken up...\n");
-                if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
-                        CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
-
-                        spin_lock_bh(&lcw_pending_timers_lock);
-                        rc = !list_empty(&lcw_pending_timers);
-                        spin_unlock_bh(&lcw_pending_timers_lock);
-                        if (rc) {
-                                CERROR("pending timers list was not empty at "
-                                       "time of watchdog dispatch shutdown\n");
-                        }
-                        break;
-                }
-
-                spin_lock_bh(&lcw_pending_timers_lock);
-                while (!list_empty(&lcw_pending_timers)) {
-
-                        lcw = list_entry(lcw_pending_timers.next,
-                                         struct lc_watchdog,
-                                         lcw_list);
-                        list_del_init(&lcw->lcw_list);
-                        spin_unlock_bh(&lcw_pending_timers_lock);
-
-                        CDEBUG(D_INFO, "found lcw for pid " LPPID ": inactive for "
-                               "%lds\n", lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time));
+                int dumplog = 1;
 
-                        if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
+               rc = wait_event_interruptible(lcw_event_waitq,
+                                             is_watchdog_fired());
+                CDEBUG(D_INFO, "Watchdog got woken up...\n");
+               if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
+                       CDEBUG(D_INFO, "LCW_FLAG_STOP set, shutting down...\n");
+
+                       spin_lock_bh(&lcw_pending_timers_lock);
+                       rc = !list_empty(&lcw_pending_timers);
+                       spin_unlock_bh(&lcw_pending_timers_lock);
+                       if (rc) {
+                               CERROR("pending timers list was not empty at "
+                                      "time of watchdog dispatch shutdown\n");
+                       }
+                       break;
+               }
+
+               spin_lock_bh(&lcw_pending_timers_lock);
+               while (!list_empty(&lcw_pending_timers)) {
+                       int is_dumplog;
+
+                       lcw = list_entry(lcw_pending_timers.next,
+                                        struct lc_watchdog, lcw_list);
+                       /* +1 ref for callback to make sure lwc wouldn't be
+                        * deleted after releasing lcw_pending_timers_lock */
+                       lcw->lcw_refcount++;
+                       spin_unlock_bh(&lcw_pending_timers_lock);
+
+                       /* lock ordering */
+                       spin_lock_bh(&lcw->lcw_lock);
+                       spin_lock_bh(&lcw_pending_timers_lock);
+
+                       if (list_empty(&lcw->lcw_list)) {
+                               /* already removed from pending list */
+                               lcw->lcw_refcount--; /* -1 ref for callback */
+                               if (lcw->lcw_refcount == 0)
+                                       list_add(&lcw->lcw_list, &zombies);
+                               spin_unlock_bh(&lcw->lcw_lock);
+                               /* still hold lcw_pending_timers_lock */
+                               continue;
+                       }
+
+                       list_del_init(&lcw->lcw_list);
+                       lcw->lcw_refcount--; /* -1 ref for pending list */
+
+                       spin_unlock_bh(&lcw_pending_timers_lock);
+                       spin_unlock_bh(&lcw->lcw_lock);
+
+                       CDEBUG(D_INFO, "found lcw for pid %d\n",
+                               lcw->lcw_pid);
+                        lcw_dump_stack(lcw);
+
+                        is_dumplog = lcw->lcw_callback == lc_watchdog_dumplog;
+                        if (lcw->lcw_state != LC_WATCHDOG_DISABLED &&
+                            (dumplog || !is_dumplog)) {
                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
+                                if (dumplog && is_dumplog)
+                                        dumplog = 0;
+                        }
 
-                        spin_lock_bh(&lcw_pending_timers_lock);
-                }
-                spin_unlock_bh(&lcw_pending_timers_lock);
-        }
+                       spin_lock_bh(&lcw_pending_timers_lock);
+                       lcw->lcw_refcount--; /* -1 ref for callback */
+                       if (lcw->lcw_refcount == 0)
+                               list_add(&lcw->lcw_list, &zombies);
+               }
+               spin_unlock_bh(&lcw_pending_timers_lock);
+
+               while (!list_empty(&zombies)) {
+                       lcw = list_entry(zombies.next,
+                                            struct lc_watchdog, lcw_list);
+                       list_del_init(&lcw->lcw_list);
+                       LIBCFS_FREE(lcw, sizeof(*lcw));
+               }
+       }
 
-        complete(&lcw_stop_completion);
+       complete(&lcw_stop_completion);
 
-        RETURN(rc);
+       RETURN(rc);
 }
 
 static void lcw_dispatch_start(void)
 {
-        int rc;
-
-        ENTRY;
-        LASSERT(lcw_refcount == 1);
-
-        init_completion(&lcw_stop_completion);
-        init_completion(&lcw_start_completion);
-        cfs_waitq_init(&lcw_event_waitq);
-
-        CDEBUG(D_INFO, "starting dispatch thread\n");
-        rc = kernel_thread(lcw_dispatch_main, NULL, 0);
-        if (rc < 0) {
-                CERROR("error spawning watchdog dispatch thread: %d\n", rc);
-                EXIT;
-                return;
-        }
-        wait_for_completion(&lcw_start_completion);
-        CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
+       struct task_struct *task;
+
+       ENTRY;
+       LASSERT(lcw_refcount == 1);
+
+       init_completion(&lcw_stop_completion);
+       init_completion(&lcw_start_completion);
+       init_waitqueue_head(&lcw_event_waitq);
+
+       CDEBUG(D_INFO, "starting dispatch thread\n");
+       task = kthread_run(lcw_dispatch_main, NULL, "lc_watchdogd");
+       if (IS_ERR(task)) {
+               CERROR("error spawning watchdog dispatch thread: %ld\n",
+                       PTR_ERR(task));
+               EXIT;
+               return;
+       }
+       wait_for_completion(&lcw_start_completion);
+       CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 
-        EXIT;
+       EXIT;
 }
 
 static void lcw_dispatch_stop(void)
 {
-        ENTRY;
-        LASSERT(lcw_refcount == 0);
+       ENTRY;
+       LASSERT(lcw_refcount == 0);
 
-        CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
+       CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 
-        set_bit(LCW_FLAG_STOP, &lcw_flags);
-        cfs_waitq_signal(&lcw_event_waitq);
+       set_bit(LCW_FLAG_STOP, &lcw_flags);
+       wake_up(&lcw_event_waitq);
 
-        wait_for_completion(&lcw_stop_completion);
+       wait_for_completion(&lcw_stop_completion);
 
-        CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
+       CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 
-        EXIT;
+       EXIT;
 }
 
-struct lc_watchdog *lc_watchdog_add(int timeout_ms,
+struct lc_watchdog *lc_watchdog_add(int timeout,
                                     void (*callback)(pid_t, void *),
                                     void *data)
 {
@@ -325,27 +350,28 @@ struct lc_watchdog *lc_watchdog_add(int timeout_ms,
                 RETURN(ERR_PTR(-ENOMEM));
         }
 
-        lcw->lcw_task     = cfs_current();
-        lcw->lcw_pid      = cfs_curproc_pid();
-        lcw->lcw_time     = cfs_time_seconds(timeout_ms) / 1000;
-        lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
-        lcw->lcw_data     = data;
-        lcw->lcw_state    = LC_WATCHDOG_DISABLED;
-
-        CFS_INIT_LIST_HEAD(&lcw->lcw_list);
-        cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw);
-
-        down(&lcw_refcount_sem);
-        if (++lcw_refcount == 1)
-                lcw_dispatch_start();
-        up(&lcw_refcount_sem);
-
-        /* Keep this working in case we enable them by default */
-        if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
-                lcw->lcw_last_touched = cfs_time_current();
-                cfs_timer_arm(&lcw->lcw_timer, lcw->lcw_time + 
-                              cfs_time_current());
-        }
+       spin_lock_init(&lcw->lcw_lock);
+       lcw->lcw_refcount = 1; /* refcount for owner */
+       lcw->lcw_task     = current;
+       lcw->lcw_pid      = current_pid();
+       lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
+       lcw->lcw_data     = data;
+       lcw->lcw_state    = LC_WATCHDOG_DISABLED;
+
+       INIT_LIST_HEAD(&lcw->lcw_list);
+       setup_timer(&lcw->lcw_timer, lcw_cb, (unsigned long)lcw);
+
+       mutex_lock(&lcw_refcount_mutex);
+       if (++lcw_refcount == 1)
+               lcw_dispatch_start();
+       mutex_unlock(&lcw_refcount_mutex);
+
+       /* Keep this working in case we enable them by default */
+       if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
+               lcw->lcw_last_touched = ktime_get();
+               mod_timer(&lcw->lcw_timer, cfs_time_seconds(timeout) +
+                         jiffies);
+       }
 
         RETURN(lcw);
 }
@@ -353,44 +379,47 @@ EXPORT_SYMBOL(lc_watchdog_add);
 
 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 {
-        cfs_time_t newtime = cfs_time_current();;
+       ktime_t newtime = ktime_get();
 
-        if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
-                struct timeval timediff;
-                cfs_time_t delta_time = cfs_time_sub(newtime,
-                                                     lcw->lcw_last_touched);
-                cfs_duration_usec(delta_time, &timediff);
+       if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
+               ktime_t lapse = ktime_sub(newtime, lcw->lcw_last_touched);
+               struct timespec64 timediff;
 
-                CWARN("Expired watchdog for pid " LPPID " %s after %lu.%.02lus\n",
-                      lcw->lcw_pid, message, timediff.tv_sec,
-                      timediff.tv_usec / 10000);
-        }
-        lcw->lcw_last_touched = newtime;
+               timediff = ktime_to_timespec64(lapse);
+               LCONSOLE_WARN("Service thread pid %u %s after %lu.%.02lus. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources).\n",
+                             lcw->lcw_pid, message,
+                             timediff.tv_sec,
+                             timediff.tv_nsec / (NSEC_PER_SEC / 100));
+       }
+       lcw->lcw_last_touched = newtime;
 }
 
-void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
+static void lc_watchdog_del_pending(struct lc_watchdog *lcw)
 {
-        ENTRY;
-        LASSERT(lcw != NULL);
+       spin_lock_bh(&lcw->lcw_lock);
+       if (unlikely(!list_empty(&lcw->lcw_list))) {
+               spin_lock_bh(&lcw_pending_timers_lock);
+               list_del_init(&lcw->lcw_list);
+               lcw->lcw_refcount--; /* -1 ref for pending list */
+               spin_unlock_bh(&lcw_pending_timers_lock);
+       }
+
+       spin_unlock_bh(&lcw->lcw_lock);
+}
 
-        spin_lock_bh(&lcw_pending_timers_lock);
-        list_del_init(&lcw->lcw_list);
-        spin_unlock_bh(&lcw_pending_timers_lock);
+void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
+{
+       ENTRY;
+       LASSERT(lcw != NULL);
 
-        lcw_update_time(lcw, "touched");
-        lcw->lcw_state = LC_WATCHDOG_ENABLED;
+       lc_watchdog_del_pending(lcw);
 
-        cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() +
-                      cfs_time_seconds(timeout_ms) / 1000);
+       lcw_update_time(lcw, "resumed");
 
-        EXIT;
-}
-EXPORT_SYMBOL(lc_watchdog_touch_ms);
+       mod_timer(&lcw->lcw_timer, jiffies + cfs_time_seconds(timeout));
+       lcw->lcw_state = LC_WATCHDOG_ENABLED;
 
-/* deprecated - use above instead */
-void lc_watchdog_touch(struct lc_watchdog *lcw)
-{
-        lc_watchdog_touch_ms(lcw, (int)cfs_duration_sec(lcw->lcw_time) * 1000);
+       EXIT;
 }
 EXPORT_SYMBOL(lc_watchdog_touch);
 
@@ -399,12 +428,9 @@ void lc_watchdog_disable(struct lc_watchdog *lcw)
         ENTRY;
         LASSERT(lcw != NULL);
 
-        spin_lock_bh(&lcw_pending_timers_lock);
-        if (!list_empty(&lcw->lcw_list))
-                list_del_init(&lcw->lcw_list);
-        spin_unlock_bh(&lcw_pending_timers_lock);
+        lc_watchdog_del_pending(lcw);
 
-        lcw_update_time(lcw, "disabled");
+        lcw_update_time(lcw, "completed");
         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 
         EXIT;
@@ -413,42 +439,42 @@ EXPORT_SYMBOL(lc_watchdog_disable);
 
 void lc_watchdog_delete(struct lc_watchdog *lcw)
 {
+        int dead;
+
         ENTRY;
         LASSERT(lcw != NULL);
 
-        cfs_timer_disarm(&lcw->lcw_timer);
+       del_timer(&lcw->lcw_timer);
 
-        lcw_update_time(lcw, "deleted");
+        lcw_update_time(lcw, "stopped");
 
-        spin_lock_bh(&lcw_pending_timers_lock);
-        if (!list_empty(&lcw->lcw_list))
-                list_del_init(&lcw->lcw_list);
-        spin_unlock_bh(&lcw_pending_timers_lock);
+       spin_lock_bh(&lcw->lcw_lock);
+       spin_lock_bh(&lcw_pending_timers_lock);
+       if (unlikely(!list_empty(&lcw->lcw_list))) {
+               list_del_init(&lcw->lcw_list);
+               lcw->lcw_refcount--; /* -1 ref for pending list */
+       }
 
-        down(&lcw_refcount_sem);
-        if (--lcw_refcount == 0)
-                lcw_dispatch_stop();
-        up(&lcw_refcount_sem);
+       lcw->lcw_refcount--; /* -1 ref for owner */
+       dead = lcw->lcw_refcount == 0;
+       spin_unlock_bh(&lcw_pending_timers_lock);
+       spin_unlock_bh(&lcw->lcw_lock);
 
-        LIBCFS_FREE(lcw, sizeof(*lcw));
+       if (dead)
+               LIBCFS_FREE(lcw, sizeof(*lcw));
 
-        EXIT;
-}
-EXPORT_SYMBOL(lc_watchdog_delete);
+       mutex_lock(&lcw_refcount_mutex);
+       if (--lcw_refcount == 0)
+               lcw_dispatch_stop();
+       mutex_unlock(&lcw_refcount_mutex);
 
-/*
- * Provided watchdog handlers
- */
-
-void lc_watchdog_dumplog(pid_t pid, void *data)
-{
-        libcfs_debug_dumplog_internal((void *)((long_ptr_t)pid));
+       EXIT;
 }
-EXPORT_SYMBOL(lc_watchdog_dumplog);
+EXPORT_SYMBOL(lc_watchdog_delete);
 
 #else   /* !defined(WITH_WATCHDOG) */
 
-struct lc_watchdog *lc_watchdog_add(int timeout_ms,
+struct lc_watchdog *lc_watchdog_add(int timeout,
                                     void (*callback)(pid_t pid, void *),
                                     void *data)
 {
@@ -457,12 +483,7 @@ struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 }
 EXPORT_SYMBOL(lc_watchdog_add);
 
-void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
-{
-}
-EXPORT_SYMBOL(lc_watchdog_touch_ms);
-
-void lc_watchdog_touch(struct lc_watchdog *lcw)
+void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 {
 }
 EXPORT_SYMBOL(lc_watchdog_touch);