libcfs/libcfs/watchdog.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * libcfs/libcfs/watchdog.c
  37  *
  38  * Author: Jacob Berkman <jacob@clusterfs.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LNET
  42
  43 #include <libcfs/libcfs.h>
  44 #include "tracefile.h"
  45
  46 struct lc_watchdog {
  47         cfs_timer_t       lcw_timer; /* kernel timer */
  48         struct list_head  lcw_list;
  49         struct timeval    lcw_last_touched;
  50         cfs_task_t       *lcw_task;
  51
  52         void            (*lcw_callback)(pid_t, void *);
  53         void             *lcw_data;
  54
  55         pid_t             lcw_pid;
  56         cfs_duration_t    lcw_time; /* time until watchdog fires, jiffies */
  57
  58         enum {
  59                 LC_WATCHDOG_DISABLED,
  60                 LC_WATCHDOG_ENABLED,
  61                 LC_WATCHDOG_EXPIRED
  62         } lcw_state;
  63 };
  64
  65 #ifdef WITH_WATCHDOG
  66 /*
  67  * The dispatcher will complete lcw_start_completion when it starts,
  68  * and lcw_stop_completion when it exits.
  69  * Wake lcw_event_waitq to signal timer callback dispatches.
  70  */
  71 static struct completion lcw_start_completion;
  72 static struct completion lcw_stop_completion;
  73 static cfs_waitq_t lcw_event_waitq;
  74
  75 /*
  76  * Set this and wake lcw_event_waitq to stop the dispatcher.
  77  */
  78 enum {
  79         LCW_FLAG_STOP = 0
  80 };
  81 static unsigned long lcw_flags = 0;
  82
  83 /*
  84  * Number of outstanding watchdogs.
  85  * When it hits 1, we start the dispatcher.
  86  * When it hits 0, we stop the distpatcher.
  87  */
  88 static __u32         lcw_refcount = 0;
  89 static DECLARE_MUTEX(lcw_refcount_sem);
  90
  91 /*
  92  * List of timers that have fired that need their callbacks run by the
  93  * dispatcher.
  94  */
  95 static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */
  96 static struct list_head lcw_pending_timers = \
  97         CFS_LIST_HEAD_INIT(lcw_pending_timers);
  98
  99 /* Last time a watchdog expired */
 100 static cfs_time_t lcw_last_watchdog_time;
 101 static int lcw_recent_watchdog_count;
 102 static spinlock_t lcw_last_watchdog_lock = SPIN_LOCK_UNLOCKED;
 103
 104 static void
 105 lcw_dump(struct lc_watchdog *lcw)
 106 {
 107         cfs_task_t *tsk;
 108 #if defined(HAVE_TASKLIST_LOCK)
 109         read_lock(&tasklist_lock);
 110 #elif defined(HAVE_TASK_RCU)
 111         rcu_read_lock();
 112 #else
 113         CERROR("unable to dump stack because of missing export\n");
 114         return;
 115 #endif
 116         ENTRY;
 117
 118         tsk = find_task_by_pid(lcw->lcw_pid);
 119
 120         if (tsk == NULL) {
 121                 CWARN("Process " LPPID " was not found in the task list; "
 122                       "watchdog callback may be incomplete\n", lcw->lcw_pid);
 123         } else if (tsk != lcw->lcw_task) {
 124                 CWARN("The current process " LPPID " did not set the watchdog; "
 125                       "watchdog callback may be incomplete\n", lcw->lcw_pid);
 126         } else {
 127                 libcfs_debug_dumpstack(tsk);
 128         }
 129
 130 #if defined(HAVE_TASKLIST_LOCK)
 131         read_unlock(&tasklist_lock);
 132 #elif defined(HAVE_TASK_RCU)
 133         rcu_read_unlock();
 134 #endif
 135         EXIT;
 136 }
 137
 138 static void lcw_cb(ulong_ptr_t data)
 139 {
 140         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 141         cfs_time_t current_time;
 142         cfs_duration_t delta_time;
 143
 144         ENTRY;
 145
 146         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 147                 EXIT;
 148                 return;
 149         }
 150
 151         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 152         current_time = cfs_time_current();
 153
 154         /* Check to see if we should throttle the watchdog timer to avoid
 155          * too many dumps going to the console thus triggering an NMI.
 156          * Normally we would not hold the spin lock over the CWARN but in
 157          * this case we hold it to ensure non ratelimited lcw_dumps are not
 158          * interleaved on the console making them hard to read. */
 159         spin_lock_bh(&lcw_last_watchdog_lock);
 160         delta_time = cfs_duration_sec(current_time - lcw_last_watchdog_time);
 161
 162         if (delta_time < libcfs_watchdog_ratelimit && lcw_recent_watchdog_count > 3) {
 163                 CWARN("Refusing to fire watchdog for pid %d: it was inactive "
 164                       "for %ldms. Rate limiting 1 per %d seconds.\n",
 165                       (int)lcw->lcw_pid,cfs_duration_sec(lcw->lcw_time) * 1000,
 166                       libcfs_watchdog_ratelimit);
 167         } else {
 168                 if (delta_time < libcfs_watchdog_ratelimit) {
 169                         lcw_recent_watchdog_count++;
 170                 } else {
 171                         memcpy(&lcw_last_watchdog_time, &current_time,
 172                                sizeof(current_time));
 173                         lcw_recent_watchdog_count = 0;
 174                 }
 175
 176                 /* This warning should appear on the console, but may not get
 177                  * into the logs since we're running in a softirq handler */
 178                 CWARN("Watchdog triggered for pid %d: it was inactive for %lds\n",
 179                       (int)lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time));
 180                 lcw_dump(lcw);
 181         }
 182
 183         spin_unlock_bh(&lcw_last_watchdog_lock);
 184         spin_lock_bh(&lcw_pending_timers_lock);
 185
 186         if (list_empty(&lcw->lcw_list)) {
 187                 list_add(&lcw->lcw_list, &lcw_pending_timers);
 188                 cfs_waitq_signal(&lcw_event_waitq);
 189         }
 190
 191         spin_unlock_bh(&lcw_pending_timers_lock);
 192
 193         EXIT;
 194 }
 195
 196 static int is_watchdog_fired(void)
 197 {
 198         int rc;
 199
 200         if (test_bit(LCW_FLAG_STOP, &lcw_flags))
 201                 return 1;
 202
 203         spin_lock_bh(&lcw_pending_timers_lock);
 204         rc = !list_empty(&lcw_pending_timers);
 205         spin_unlock_bh(&lcw_pending_timers_lock);
 206         return rc;
 207 }
 208
 209 static int lcw_dispatch_main(void *data)
 210 {
 211         int                 rc = 0;
 212         unsigned long       flags;
 213         struct lc_watchdog *lcw;
 214
 215         ENTRY;
 216
 217         cfs_daemonize("lc_watchdogd");
 218
 219         SIGNAL_MASK_LOCK(current, flags);
 220         sigfillset(&current->blocked);
 221         RECALC_SIGPENDING;
 222         SIGNAL_MASK_UNLOCK(current, flags);
 223
 224         complete(&lcw_start_completion);
 225
 226         while (1) {
 227                 cfs_wait_event_interruptible(lcw_event_waitq, is_watchdog_fired(), rc);
 228                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 229                 if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 230                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 231
 232                         spin_lock_bh(&lcw_pending_timers_lock);
 233                         rc = !list_empty(&lcw_pending_timers);
 234                         spin_unlock_bh(&lcw_pending_timers_lock);
 235                         if (rc) {
 236                                 CERROR("pending timers list was not empty at "
 237                                        "time of watchdog dispatch shutdown\n");
 238                         }
 239                         break;
 240                 }
 241
 242                 spin_lock_bh(&lcw_pending_timers_lock);
 243                 while (!list_empty(&lcw_pending_timers)) {
 244
 245                         lcw = list_entry(lcw_pending_timers.next,
 246                                          struct lc_watchdog,
 247                                          lcw_list);
 248                         list_del_init(&lcw->lcw_list);
 249                         spin_unlock_bh(&lcw_pending_timers_lock);
 250
 251                         CDEBUG(D_INFO, "found lcw for pid " LPPID ": inactive for "
 252                                "%lds\n", lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time));
 253
 254                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
 255                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 256
 257                         spin_lock_bh(&lcw_pending_timers_lock);
 258                 }
 259                 spin_unlock_bh(&lcw_pending_timers_lock);
 260         }
 261
 262         complete(&lcw_stop_completion);
 263
 264         RETURN(rc);
 265 }
 266
 267 static void lcw_dispatch_start(void)
 268 {
 269         int rc;
 270
 271         ENTRY;
 272         LASSERT(lcw_refcount == 1);
 273
 274         init_completion(&lcw_stop_completion);
 275         init_completion(&lcw_start_completion);
 276         cfs_waitq_init(&lcw_event_waitq);
 277
 278         CDEBUG(D_INFO, "starting dispatch thread\n");
 279         rc = kernel_thread(lcw_dispatch_main, NULL, 0);
 280         if (rc < 0) {
 281                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 282                 EXIT;
 283                 return;
 284         }
 285         wait_for_completion(&lcw_start_completion);
 286         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 287
 288         EXIT;
 289 }
 290
 291 static void lcw_dispatch_stop(void)
 292 {
 293         ENTRY;
 294         LASSERT(lcw_refcount == 0);
 295
 296         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 297
 298         set_bit(LCW_FLAG_STOP, &lcw_flags);
 299         cfs_waitq_signal(&lcw_event_waitq);
 300
 301         wait_for_completion(&lcw_stop_completion);
 302
 303         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 304
 305         EXIT;
 306 }
 307
 308 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 309                                     void (*callback)(pid_t, void *),
 310                                     void *data)
 311 {
 312         struct lc_watchdog *lcw = NULL;
 313         ENTRY;
 314
 315         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 316         if (lcw == NULL) {
 317                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 318                 RETURN(ERR_PTR(-ENOMEM));
 319         }
 320
 321         lcw->lcw_task     = cfs_current();
 322         lcw->lcw_pid      = cfs_curproc_pid();
 323         lcw->lcw_time     = cfs_time_seconds(timeout_ms) / 1000;
 324         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 325         lcw->lcw_data     = data;
 326         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 327
 328         CFS_INIT_LIST_HEAD(&lcw->lcw_list);
 329         cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw);
 330
 331         down(&lcw_refcount_sem);
 332         if (++lcw_refcount == 1)
 333                 lcw_dispatch_start();
 334         up(&lcw_refcount_sem);
 335
 336         /* Keep this working in case we enable them by default */
 337         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 338                 do_gettimeofday(&lcw->lcw_last_touched);
 339                 cfs_timer_arm(&lcw->lcw_timer, lcw->lcw_time +
 340                               cfs_time_current());
 341         }
 342
 343         RETURN(lcw);
 344 }
 345 EXPORT_SYMBOL(lc_watchdog_add);
 346
 347 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 348 {
 349         struct timeval newtime;
 350         struct timeval timediff;
 351
 352         do_gettimeofday(&newtime);
 353         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 354                 cfs_timeval_sub(&newtime, &lcw->lcw_last_touched, &timediff);
 355                 CWARN("Expired watchdog for pid " LPPID " %s after %lu.%.4lus\n",
 356                       lcw->lcw_pid,
 357                       message,
 358                       timediff.tv_sec,
 359                       timediff.tv_usec / 100);
 360         }
 361         lcw->lcw_last_touched = newtime;
 362 }
 363
 364 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 365 {
 366         ENTRY;
 367         LASSERT(lcw != NULL);
 368
 369         spin_lock_bh(&lcw_pending_timers_lock);
 370         list_del_init(&lcw->lcw_list);
 371         spin_unlock_bh(&lcw_pending_timers_lock);
 372
 373         lcw_update_time(lcw, "touched");
 374         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 375
 376         cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() +
 377                       cfs_time_seconds(timeout_ms) / 1000);
 378
 379         EXIT;
 380 }
 381 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 382
 383 /* deprecated - use above instead */
 384 void lc_watchdog_touch(struct lc_watchdog *lcw)
 385 {
 386         lc_watchdog_touch_ms(lcw, (int)cfs_duration_sec(lcw->lcw_time) * 1000);
 387 }
 388 EXPORT_SYMBOL(lc_watchdog_touch);
 389
 390 void lc_watchdog_disable(struct lc_watchdog *lcw)
 391 {
 392         ENTRY;
 393         LASSERT(lcw != NULL);
 394
 395         spin_lock_bh(&lcw_pending_timers_lock);
 396         if (!list_empty(&lcw->lcw_list))
 397                 list_del_init(&lcw->lcw_list);
 398         spin_unlock_bh(&lcw_pending_timers_lock);
 399
 400         lcw_update_time(lcw, "disabled");
 401         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 402
 403         EXIT;
 404 }
 405 EXPORT_SYMBOL(lc_watchdog_disable);
 406
 407 void lc_watchdog_delete(struct lc_watchdog *lcw)
 408 {
 409         ENTRY;
 410         LASSERT(lcw != NULL);
 411
 412         cfs_timer_disarm(&lcw->lcw_timer);
 413
 414         lcw_update_time(lcw, "deleted");
 415
 416         spin_lock_bh(&lcw_pending_timers_lock);
 417         if (!list_empty(&lcw->lcw_list))
 418                 list_del_init(&lcw->lcw_list);
 419         spin_unlock_bh(&lcw_pending_timers_lock);
 420
 421         down(&lcw_refcount_sem);
 422         if (--lcw_refcount == 0)
 423                 lcw_dispatch_stop();
 424         up(&lcw_refcount_sem);
 425
 426         LIBCFS_FREE(lcw, sizeof(*lcw));
 427
 428         EXIT;
 429 }
 430 EXPORT_SYMBOL(lc_watchdog_delete);
 431
 432 /*
 433  * Provided watchdog handlers
 434  */
 435
 436 void lc_watchdog_dumplog(pid_t pid, void *data)
 437 {
 438         libcfs_debug_dumplog_internal((void *)((long_ptr_t)pid));
 439 }
 440 EXPORT_SYMBOL(lc_watchdog_dumplog);
 441
 442 #else   /* !defined(WITH_WATCHDOG) */
 443
 444 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 445                                     void (*callback)(pid_t pid, void *),
 446                                     void *data)
 447 {
 448         static struct lc_watchdog      watchdog;
 449         return &watchdog;
 450 }
 451 EXPORT_SYMBOL(lc_watchdog_add);
 452
 453 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 454 {
 455 }
 456 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 457
 458 void lc_watchdog_touch(struct lc_watchdog *lcw)
 459 {
 460 }
 461 EXPORT_SYMBOL(lc_watchdog_touch);
 462
 463 void lc_watchdog_disable(struct lc_watchdog *lcw)
 464 {
 465 }
 466 EXPORT_SYMBOL(lc_watchdog_disable);
 467
 468 void lc_watchdog_delete(struct lc_watchdog *lcw)
 469 {
 470 }
 471 EXPORT_SYMBOL(lc_watchdog_delete);
 472
 473 #endif