libcfs/libcfs/watchdog.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * libcfs/libcfs/watchdog.c
  37  *
  38  * Author: Jacob Berkman <jacob@clusterfs.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LNET
  42
  43 #include <libcfs/libcfs.h>
  44 #include "tracefile.h"
  45
  46 struct lc_watchdog {
  47         cfs_timer_t       lcw_timer; /* kernel timer */
  48         struct list_head  lcw_list;
  49         cfs_time_t        lcw_last_touched;
  50         cfs_task_t       *lcw_task;
  51
  52         void            (*lcw_callback)(pid_t, void *);
  53         void             *lcw_data;
  54
  55         pid_t             lcw_pid;
  56
  57         enum {
  58                 LC_WATCHDOG_DISABLED,
  59                 LC_WATCHDOG_ENABLED,
  60                 LC_WATCHDOG_EXPIRED
  61         } lcw_state;
  62 };
  63
  64 #ifdef WITH_WATCHDOG
  65 /*
  66  * The dispatcher will complete lcw_start_completion when it starts,
  67  * and lcw_stop_completion when it exits.
  68  * Wake lcw_event_waitq to signal timer callback dispatches.
  69  */
  70 static struct completion lcw_start_completion;
  71 static struct completion lcw_stop_completion;
  72 static cfs_waitq_t lcw_event_waitq;
  73
  74 /*
  75  * Set this and wake lcw_event_waitq to stop the dispatcher.
  76  */
  77 enum {
  78         LCW_FLAG_STOP = 0
  79 };
  80 static unsigned long lcw_flags = 0;
  81
  82 /*
  83  * Number of outstanding watchdogs.
  84  * When it hits 1, we start the dispatcher.
  85  * When it hits 0, we stop the distpatcher.
  86  */
  87 static __u32         lcw_refcount = 0;
  88 static DECLARE_MUTEX(lcw_refcount_sem);
  89
  90 /*
  91  * List of timers that have fired that need their callbacks run by the
  92  * dispatcher.
  93  */
  94 static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */
  95 static struct list_head lcw_pending_timers = \
  96         CFS_LIST_HEAD_INIT(lcw_pending_timers);
  97
  98 /* Last time a watchdog expired */
  99 static cfs_time_t lcw_last_watchdog_time;
 100 static int lcw_recent_watchdog_count;
 101 static spinlock_t lcw_last_watchdog_lock = SPIN_LOCK_UNLOCKED;
 102
 103 static void
 104 lcw_dump(struct lc_watchdog *lcw)
 105 {
 106         cfs_task_t *tsk;
 107 #if defined(HAVE_TASKLIST_LOCK)
 108         read_lock(&tasklist_lock);
 109 #elif defined(HAVE_TASK_RCU)
 110         rcu_read_lock();
 111 #else
 112         CERROR("unable to dump stack because of missing export\n");
 113         return;
 114 #endif
 115         ENTRY;
 116
 117         tsk = find_task_by_pid(lcw->lcw_pid);
 118
 119         if (tsk == NULL) {
 120                 CWARN("Process " LPPID " was not found in the task list; "
 121                       "watchdog callback may be incomplete\n", lcw->lcw_pid);
 122         } else if (tsk != lcw->lcw_task) {
 123                 CWARN("The current process " LPPID " did not set the watchdog; "
 124                       "watchdog callback may be incomplete\n", lcw->lcw_pid);
 125         } else {
 126                 libcfs_debug_dumpstack(tsk);
 127         }
 128
 129 #if defined(HAVE_TASKLIST_LOCK)
 130         read_unlock(&tasklist_lock);
 131 #elif defined(HAVE_TASK_RCU)
 132         rcu_read_unlock();
 133 #endif
 134         EXIT;
 135 }
 136
 137 static void lcw_cb(ulong_ptr_t data)
 138 {
 139         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 140         cfs_time_t current_time;
 141         cfs_duration_t delta_time;
 142         struct timeval timediff;
 143
 144         ENTRY;
 145
 146         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 147                 EXIT;
 148                 return;
 149         }
 150
 151         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 152         current_time = cfs_time_current();
 153
 154         delta_time = cfs_time_sub(current_time, lcw->lcw_last_touched);
 155         cfs_duration_usec(delta_time, &timediff);
 156
 157         /* Check to see if we should throttle the watchdog timer to avoid
 158          * too many dumps going to the console thus triggering an NMI.
 159          * Normally we would not hold the spin lock over the CWARN but in
 160          * this case we hold it to ensure non ratelimited lcw_dumps are not
 161          * interleaved on the console making them hard to read. */
 162         spin_lock_bh(&lcw_last_watchdog_lock);
 163         delta_time = cfs_duration_sec(cfs_time_sub(current_time,
 164                                                    lcw_last_watchdog_time));
 165
 166         if (delta_time < libcfs_watchdog_ratelimit &&
 167             lcw_recent_watchdog_count > 3) {
 168                 CWARN("Refusing to fire watchdog for pid %d: it was inactive "
 169                       "for %lu.%.02lus. Rate limiting 1 per %d seconds.\n",
 170                       (int)lcw->lcw_pid, timediff.tv_sec,
 171                       timediff.tv_usec / 10000, libcfs_watchdog_ratelimit);
 172         } else {
 173                 if (delta_time < libcfs_watchdog_ratelimit) {
 174                         lcw_recent_watchdog_count++;
 175                 } else {
 176                         memcpy(&lcw_last_watchdog_time, &current_time,
 177                                sizeof(current_time));
 178                         lcw_recent_watchdog_count = 0;
 179                 }
 180
 181                 /* This warning should appear on the console, but may not get
 182                  * into the logs since we're running in a softirq handler */
 183                 CWARN("Watchdog triggered for pid %d: it was inactive for "
 184                       "%lu.%.02lus\n", (int)lcw->lcw_pid, timediff.tv_sec,
 185                       timediff.tv_usec / 10000);
 186                 lcw_dump(lcw);
 187         }
 188
 189         spin_unlock_bh(&lcw_last_watchdog_lock);
 190         spin_lock_bh(&lcw_pending_timers_lock);
 191
 192         if (list_empty(&lcw->lcw_list)) {
 193                 list_add(&lcw->lcw_list, &lcw_pending_timers);
 194                 cfs_waitq_signal(&lcw_event_waitq);
 195         }
 196
 197         spin_unlock_bh(&lcw_pending_timers_lock);
 198
 199         EXIT;
 200 }
 201
 202 static int is_watchdog_fired(void)
 203 {
 204         int rc;
 205
 206         if (test_bit(LCW_FLAG_STOP, &lcw_flags))
 207                 return 1;
 208
 209         spin_lock_bh(&lcw_pending_timers_lock);
 210         rc = !list_empty(&lcw_pending_timers);
 211         spin_unlock_bh(&lcw_pending_timers_lock);
 212         return rc;
 213 }
 214
 215 static int lcw_dispatch_main(void *data)
 216 {
 217         int                 rc = 0;
 218         unsigned long       flags;
 219         struct lc_watchdog *lcw;
 220
 221         ENTRY;
 222
 223         cfs_daemonize("lc_watchdogd");
 224
 225         SIGNAL_MASK_LOCK(current, flags);
 226         sigfillset(&current->blocked);
 227         RECALC_SIGPENDING;
 228         SIGNAL_MASK_UNLOCK(current, flags);
 229
 230         complete(&lcw_start_completion);
 231
 232         while (1) {
 233                 cfs_wait_event_interruptible(lcw_event_waitq, is_watchdog_fired(), rc);
 234                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 235                 if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 236                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 237
 238                         spin_lock_bh(&lcw_pending_timers_lock);
 239                         rc = !list_empty(&lcw_pending_timers);
 240                         spin_unlock_bh(&lcw_pending_timers_lock);
 241                         if (rc) {
 242                                 CERROR("pending timers list was not empty at "
 243                                        "time of watchdog dispatch shutdown\n");
 244                         }
 245                         break;
 246                 }
 247
 248                 spin_lock_bh(&lcw_pending_timers_lock);
 249                 while (!list_empty(&lcw_pending_timers)) {
 250
 251                         lcw = list_entry(lcw_pending_timers.next,
 252                                          struct lc_watchdog,
 253                                          lcw_list);
 254                         list_del_init(&lcw->lcw_list);
 255                         spin_unlock_bh(&lcw_pending_timers_lock);
 256
 257                         CDEBUG(D_INFO, "found lcw for pid " LPPID "\n", lcw->lcw_pid);
 258
 259                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
 260                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 261
 262                         spin_lock_bh(&lcw_pending_timers_lock);
 263                 }
 264                 spin_unlock_bh(&lcw_pending_timers_lock);
 265         }
 266
 267         complete(&lcw_stop_completion);
 268
 269         RETURN(rc);
 270 }
 271
 272 static void lcw_dispatch_start(void)
 273 {
 274         int rc;
 275
 276         ENTRY;
 277         LASSERT(lcw_refcount == 1);
 278
 279         init_completion(&lcw_stop_completion);
 280         init_completion(&lcw_start_completion);
 281         cfs_waitq_init(&lcw_event_waitq);
 282
 283         CDEBUG(D_INFO, "starting dispatch thread\n");
 284         rc = kernel_thread(lcw_dispatch_main, NULL, 0);
 285         if (rc < 0) {
 286                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 287                 EXIT;
 288                 return;
 289         }
 290         wait_for_completion(&lcw_start_completion);
 291         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 292
 293         EXIT;
 294 }
 295
 296 static void lcw_dispatch_stop(void)
 297 {
 298         ENTRY;
 299         LASSERT(lcw_refcount == 0);
 300
 301         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 302
 303         set_bit(LCW_FLAG_STOP, &lcw_flags);
 304         cfs_waitq_signal(&lcw_event_waitq);
 305
 306         wait_for_completion(&lcw_stop_completion);
 307
 308         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 309
 310         EXIT;
 311 }
 312
 313 struct lc_watchdog *lc_watchdog_add(int timeout,
 314                                     void (*callback)(pid_t, void *),
 315                                     void *data)
 316 {
 317         struct lc_watchdog *lcw = NULL;
 318         ENTRY;
 319
 320         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 321         if (lcw == NULL) {
 322                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 323                 RETURN(ERR_PTR(-ENOMEM));
 324         }
 325
 326         lcw->lcw_task     = cfs_current();
 327         lcw->lcw_pid      = cfs_curproc_pid();
 328         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 329         lcw->lcw_data     = data;
 330         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 331
 332         CFS_INIT_LIST_HEAD(&lcw->lcw_list);
 333         cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw);
 334
 335         down(&lcw_refcount_sem);
 336         if (++lcw_refcount == 1)
 337                 lcw_dispatch_start();
 338         up(&lcw_refcount_sem);
 339
 340         /* Keep this working in case we enable them by default */
 341         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 342                 lcw->lcw_last_touched = cfs_time_current();
 343                 cfs_timer_arm(&lcw->lcw_timer, cfs_time_seconds(timeout) +
 344                               cfs_time_current());
 345         }
 346
 347         RETURN(lcw);
 348 }
 349 EXPORT_SYMBOL(lc_watchdog_add);
 350
 351 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 352 {
 353         cfs_time_t newtime = cfs_time_current();;
 354
 355         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 356                 struct timeval timediff;
 357                 cfs_time_t delta_time = cfs_time_sub(newtime,
 358                                                      lcw->lcw_last_touched);
 359                 cfs_duration_usec(delta_time, &timediff);
 360
 361                 CWARN("Expired watchdog for pid " LPPID " %s after %lu.%.02lus\n",
 362                       lcw->lcw_pid, message, timediff.tv_sec,
 363                       timediff.tv_usec / 10000);
 364         }
 365         lcw->lcw_last_touched = newtime;
 366 }
 367
 368 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 369 {
 370         ENTRY;
 371         LASSERT(lcw != NULL);
 372
 373         spin_lock_bh(&lcw_pending_timers_lock);
 374         list_del_init(&lcw->lcw_list);
 375         spin_unlock_bh(&lcw_pending_timers_lock);
 376
 377         lcw_update_time(lcw, "touched");
 378         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 379
 380         cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() +
 381                       cfs_time_seconds(timeout));
 382
 383         EXIT;
 384 }
 385 EXPORT_SYMBOL(lc_watchdog_touch);
 386
 387 void lc_watchdog_disable(struct lc_watchdog *lcw)
 388 {
 389         ENTRY;
 390         LASSERT(lcw != NULL);
 391
 392         spin_lock_bh(&lcw_pending_timers_lock);
 393         if (!list_empty(&lcw->lcw_list))
 394                 list_del_init(&lcw->lcw_list);
 395         spin_unlock_bh(&lcw_pending_timers_lock);
 396
 397         lcw_update_time(lcw, "disabled");
 398         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 399
 400         EXIT;
 401 }
 402 EXPORT_SYMBOL(lc_watchdog_disable);
 403
 404 void lc_watchdog_delete(struct lc_watchdog *lcw)
 405 {
 406         ENTRY;
 407         LASSERT(lcw != NULL);
 408
 409         cfs_timer_disarm(&lcw->lcw_timer);
 410
 411         lcw_update_time(lcw, "deleted");
 412
 413         spin_lock_bh(&lcw_pending_timers_lock);
 414         if (!list_empty(&lcw->lcw_list))
 415                 list_del_init(&lcw->lcw_list);
 416         spin_unlock_bh(&lcw_pending_timers_lock);
 417
 418         down(&lcw_refcount_sem);
 419         if (--lcw_refcount == 0)
 420                 lcw_dispatch_stop();
 421         up(&lcw_refcount_sem);
 422
 423         LIBCFS_FREE(lcw, sizeof(*lcw));
 424
 425         EXIT;
 426 }
 427 EXPORT_SYMBOL(lc_watchdog_delete);
 428
 429 /*
 430  * Provided watchdog handlers
 431  */
 432
 433 void lc_watchdog_dumplog(pid_t pid, void *data)
 434 {
 435         libcfs_debug_dumplog_internal((void *)((long_ptr_t)pid));
 436 }
 437 EXPORT_SYMBOL(lc_watchdog_dumplog);
 438
 439 #else   /* !defined(WITH_WATCHDOG) */
 440
 441 struct lc_watchdog *lc_watchdog_add(int timeout,
 442                                     void (*callback)(pid_t pid, void *),
 443                                     void *data)
 444 {
 445         static struct lc_watchdog      watchdog;
 446         return &watchdog;
 447 }
 448 EXPORT_SYMBOL(lc_watchdog_add);
 449
 450 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 451 {
 452 }
 453 EXPORT_SYMBOL(lc_watchdog_touch);
 454
 455 void lc_watchdog_disable(struct lc_watchdog *lcw)
 456 {
 457 }
 458 EXPORT_SYMBOL(lc_watchdog_disable);
 459
 460 void lc_watchdog_delete(struct lc_watchdog *lcw)
 461 {
 462 }
 463 EXPORT_SYMBOL(lc_watchdog_delete);
 464
 465 #endif