lnet/libcfs/watchdog.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/libcfs/watchdog.c
  37  *
  38  * Author: Jacob Berkman <jacob@clusterfs.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LNET
  42
  43 #include <libcfs/kp30.h>
  44 #include <libcfs/libcfs.h>
  45 #include "tracefile.h"
  46
  47 struct lc_watchdog {
  48         cfs_timer_t       lcw_timer; /* kernel timer */
  49         struct list_head  lcw_list;
  50         cfs_time_t        lcw_last_touched;
  51         cfs_task_t       *lcw_task;
  52
  53         void            (*lcw_callback)(pid_t, void *);
  54         void             *lcw_data;
  55
  56         pid_t             lcw_pid;
  57
  58         enum {
  59                 LC_WATCHDOG_DISABLED,
  60                 LC_WATCHDOG_ENABLED,
  61                 LC_WATCHDOG_EXPIRED
  62         } lcw_state;
  63 };
  64
  65 #ifdef WITH_WATCHDOG
  66 /*
  67  * The dispatcher will complete lcw_start_completion when it starts,
  68  * and lcw_stop_completion when it exits.
  69  * Wake lcw_event_waitq to signal timer callback dispatches.
  70  */
  71 static struct completion lcw_start_completion;
  72 static struct completion lcw_stop_completion;
  73 static wait_queue_head_t lcw_event_waitq;
  74
  75 /*
  76  * Set this and wake lcw_event_waitq to stop the dispatcher.
  77  */
  78 enum {
  79         LCW_FLAG_STOP = 0
  80 };
  81 static unsigned long lcw_flags = 0;
  82
  83 /*
  84  * Number of outstanding watchdogs.
  85  * When it hits 1, we start the dispatcher.
  86  * When it hits 0, we stop the distpatcher.
  87  */
  88 static __u32         lcw_refcount = 0;
  89 static DECLARE_MUTEX(lcw_refcount_sem);
  90
  91 /*
  92  * List of timers that have fired that need their callbacks run by the
  93  * dispatcher.
  94  */
  95 static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */
  96 static struct list_head lcw_pending_timers = \
  97         LIST_HEAD_INIT(lcw_pending_timers);
  98
  99 /* Last time a watchdog expired */
 100 static cfs_time_t lcw_last_watchdog_time;
 101 static int lcw_recent_watchdog_count;
 102 static spinlock_t lcw_last_watchdog_lock = SPIN_LOCK_UNLOCKED;
 103
 104 static void
 105 lcw_dump(struct lc_watchdog *lcw)
 106 {
 107         ENTRY;
 108
 109 #if defined(HAVE_TASKLIST_LOCK)
 110         read_lock(&tasklist_lock);
 111 #elif defined(HAVE_TASK_RCU)
 112         rcu_read_lock();
 113 #else
 114         CERROR("unable to dump stack because of missing export\n");
 115         RETURN_EXIT;
 116 #endif
 117         if (lcw->lcw_task == NULL) {
 118                 LCONSOLE_WARN("Process %d was not found in the task list; "
 119                               "watchdog callback may be incomplete\n",
 120                               (int)lcw->lcw_pid);
 121         } else {
 122                 libcfs_debug_dumpstack(lcw->lcw_task);
 123         }
 124
 125 #if defined(HAVE_TASKLIST_LOCK)
 126         read_unlock(&tasklist_lock);
 127 #elif defined(HAVE_TASK_RCU)
 128         rcu_read_unlock();
 129 #endif
 130         EXIT;
 131 }
 132
 133 static void lcw_cb(unsigned long data)
 134 {
 135         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 136         cfs_time_t current_time;
 137         cfs_duration_t delta_time;
 138         struct timeval timediff;
 139
 140         ENTRY;
 141
 142         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 143                 EXIT;
 144                 return;
 145         }
 146
 147         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 148         current_time = cfs_time_current();
 149
 150         delta_time = cfs_time_sub(current_time, lcw->lcw_last_touched);
 151         cfs_duration_usec(delta_time, &timediff);
 152
 153         /* Check to see if we should throttle the watchdog timer to avoid
 154          * too many dumps going to the console thus triggering an NMI.
 155          * Normally we would not hold the spin lock over the CWARN but in
 156          * this case we hold it to ensure non ratelimited lcw_dumps are not
 157          * interleaved on the console making them hard to read. */
 158         spin_lock_bh(&lcw_last_watchdog_lock);
 159         delta_time = cfs_duration_sec(cfs_time_sub(current_time,
 160                                                    lcw_last_watchdog_time));
 161
 162         if (delta_time < libcfs_watchdog_ratelimit &&
 163             lcw_recent_watchdog_count > 3) {
 164                 LCONSOLE_WARN("Service thread pid %u was inactive for "
 165                               "%lu.%.02lus. Watchdog stack traces are limited "
 166                               "to 3 per %d seconds, skipping this one.\n",
 167                               (int)lcw->lcw_pid,
 168                               timediff.tv_sec,
 169                               timediff.tv_usec / 10000,
 170                               libcfs_watchdog_ratelimit);
 171         } else {
 172                 if (delta_time < libcfs_watchdog_ratelimit) {
 173                         lcw_recent_watchdog_count++;
 174                 } else {
 175                         memcpy(&lcw_last_watchdog_time, &current_time,
 176                                sizeof(current_time));
 177                         lcw_recent_watchdog_count = 0;
 178                 }
 179
 180                 /* This warning should appear on the console, but may not get
 181                  * into the logs since we're running in a softirq handler */
 182                 LCONSOLE_WARN("Service thread pid %u was inactive for "
 183                               "%lu.%.02lus. The thread might be hung, or it "
 184                               "might only be slow and will resume later. "
 185                               "Dumping the stack trace for debugging purposes:"
 186                               "\n",
 187                               (int)lcw->lcw_pid,
 188                               timediff.tv_sec,
 189                               timediff.tv_usec / 10000);
 190                 lcw_dump(lcw);
 191         }
 192
 193         spin_unlock_bh(&lcw_last_watchdog_lock);
 194         spin_lock_bh(&lcw_pending_timers_lock);
 195
 196         if (list_empty(&lcw->lcw_list)) {
 197                 list_add(&lcw->lcw_list, &lcw_pending_timers);
 198                 wake_up(&lcw_event_waitq);
 199         }
 200
 201         spin_unlock_bh(&lcw_pending_timers_lock);
 202
 203         EXIT;
 204 }
 205
 206 static int is_watchdog_fired(void)
 207 {
 208         int rc;
 209
 210         if (test_bit(LCW_FLAG_STOP, &lcw_flags))
 211                 return 1;
 212
 213         spin_lock_bh(&lcw_pending_timers_lock);
 214         rc = !list_empty(&lcw_pending_timers);
 215         spin_unlock_bh(&lcw_pending_timers_lock);
 216         return rc;
 217 }
 218
 219 static int lcw_dispatch_main(void *data)
 220 {
 221         int                 rc = 0;
 222         unsigned long       flags;
 223         struct lc_watchdog *lcw;
 224
 225         ENTRY;
 226
 227         cfs_daemonize("lc_watchdogd");
 228
 229         SIGNAL_MASK_LOCK(current, flags);
 230         sigfillset(&current->blocked);
 231         RECALC_SIGPENDING;
 232         SIGNAL_MASK_UNLOCK(current, flags);
 233
 234         complete(&lcw_start_completion);
 235
 236         while (1) {
 237                 wait_event_interruptible(lcw_event_waitq, is_watchdog_fired());
 238                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 239                 if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 240                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 241
 242                         spin_lock_bh(&lcw_pending_timers_lock);
 243                         rc = !list_empty(&lcw_pending_timers);
 244                         spin_unlock_bh(&lcw_pending_timers_lock);
 245                         if (rc) {
 246                                 CERROR("pending timers list was not empty at "
 247                                        "time of watchdog dispatch shutdown\n");
 248                         }
 249                         break;
 250                 }
 251
 252                 spin_lock_bh(&lcw_pending_timers_lock);
 253                 while (!list_empty(&lcw_pending_timers)) {
 254
 255                         lcw = list_entry(lcw_pending_timers.next,
 256                                          struct lc_watchdog,
 257                                          lcw_list);
 258                         list_del_init(&lcw->lcw_list);
 259                         spin_unlock_bh(&lcw_pending_timers_lock);
 260
 261                         CDEBUG(D_INFO, "found lcw for pid %d\n",
 262                                (int)lcw->lcw_pid);
 263
 264                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
 265                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 266
 267                         spin_lock_bh(&lcw_pending_timers_lock);
 268                 }
 269                 spin_unlock_bh(&lcw_pending_timers_lock);
 270         }
 271
 272         complete(&lcw_stop_completion);
 273
 274         RETURN(rc);
 275 }
 276
 277 static void lcw_dispatch_start(void)
 278 {
 279         int rc;
 280
 281         ENTRY;
 282         LASSERT(lcw_refcount == 1);
 283
 284         init_completion(&lcw_stop_completion);
 285         init_completion(&lcw_start_completion);
 286         init_waitqueue_head(&lcw_event_waitq);
 287
 288         CDEBUG(D_INFO, "starting dispatch thread\n");
 289         rc = kernel_thread(lcw_dispatch_main, NULL, 0);
 290         if (rc < 0) {
 291                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 292                 EXIT;
 293                 return;
 294         }
 295         wait_for_completion(&lcw_start_completion);
 296         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 297
 298         EXIT;
 299 }
 300
 301 static void lcw_dispatch_stop(void)
 302 {
 303         ENTRY;
 304         LASSERT(lcw_refcount == 0);
 305
 306         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 307
 308         set_bit(LCW_FLAG_STOP, &lcw_flags);
 309         wake_up(&lcw_event_waitq);
 310
 311         wait_for_completion(&lcw_stop_completion);
 312
 313         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 314
 315         EXIT;
 316 }
 317
 318 struct lc_watchdog *lc_watchdog_add(int timeout,
 319                                     void (*callback)(pid_t, void *),
 320                                     void *data)
 321 {
 322         struct lc_watchdog *lcw = NULL;
 323         ENTRY;
 324
 325         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 326         if (lcw == NULL) {
 327                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 328                 RETURN(ERR_PTR(-ENOMEM));
 329         }
 330
 331         lcw->lcw_task     = cfs_current();
 332         lcw->lcw_pid      = cfs_curproc_pid();
 333         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 334         lcw->lcw_data     = data;
 335         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 336
 337         INIT_LIST_HEAD(&lcw->lcw_list);
 338
 339         lcw->lcw_timer.function = lcw_cb;
 340         lcw->lcw_timer.data = (unsigned long)lcw;
 341         lcw->lcw_timer.expires = jiffies + cfs_time_seconds(timeout);
 342         init_timer(&lcw->lcw_timer);
 343
 344         down(&lcw_refcount_sem);
 345         if (++lcw_refcount == 1)
 346                 lcw_dispatch_start();
 347         up(&lcw_refcount_sem);
 348
 349         /* Keep this working in case we enable them by default */
 350         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 351                 lcw->lcw_last_touched = cfs_time_current();
 352                 add_timer(&lcw->lcw_timer);
 353         }
 354
 355         RETURN(lcw);
 356 }
 357 EXPORT_SYMBOL(lc_watchdog_add);
 358
 359 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 360 {
 361         cfs_time_t newtime = cfs_time_current();;
 362
 363         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 364                 struct timeval timediff;
 365                 cfs_time_t delta_time = cfs_time_sub(newtime,
 366                                                      lcw->lcw_last_touched);
 367                 cfs_duration_usec(delta_time, &timediff);
 368
 369                 LCONSOLE_WARN("Service thread pid %u %s after %lu.%.02lus. "
 370                               "This indicates the system was overloaded (too "
 371                               "many service threads, or there were not enough "
 372                               "hardware resources).\n",
 373                               lcw->lcw_pid,
 374                               message,
 375                               timediff.tv_sec,
 376                               timediff.tv_usec / 10000);
 377         }
 378         lcw->lcw_last_touched = newtime;
 379 }
 380
 381 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 382 {
 383         ENTRY;
 384         LASSERT(lcw != NULL);
 385
 386         spin_lock_bh(&lcw_pending_timers_lock);
 387         list_del_init(&lcw->lcw_list);
 388         spin_unlock_bh(&lcw_pending_timers_lock);
 389
 390         lcw_update_time(lcw, "resumed");
 391         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 392
 393         mod_timer(&lcw->lcw_timer, jiffies + cfs_time_seconds(timeout));
 394
 395         EXIT;
 396 }
 397 EXPORT_SYMBOL(lc_watchdog_touch);
 398
 399 void lc_watchdog_disable(struct lc_watchdog *lcw)
 400 {
 401         ENTRY;
 402         LASSERT(lcw != NULL);
 403
 404         spin_lock_bh(&lcw_pending_timers_lock);
 405         if (!list_empty(&lcw->lcw_list))
 406                 list_del_init(&lcw->lcw_list);
 407         spin_unlock_bh(&lcw_pending_timers_lock);
 408
 409         lcw_update_time(lcw, "completed");
 410         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 411
 412         EXIT;
 413 }
 414 EXPORT_SYMBOL(lc_watchdog_disable);
 415
 416 void lc_watchdog_delete(struct lc_watchdog *lcw)
 417 {
 418         ENTRY;
 419         LASSERT(lcw != NULL);
 420
 421         del_timer(&lcw->lcw_timer);
 422
 423         lcw_update_time(lcw, "stopped");
 424
 425         spin_lock_bh(&lcw_pending_timers_lock);
 426         if (!list_empty(&lcw->lcw_list))
 427                 list_del_init(&lcw->lcw_list);
 428         spin_unlock_bh(&lcw_pending_timers_lock);
 429
 430         down(&lcw_refcount_sem);
 431         if (--lcw_refcount == 0)
 432                 lcw_dispatch_stop();
 433         up(&lcw_refcount_sem);
 434
 435         LIBCFS_FREE(lcw, sizeof(*lcw));
 436
 437         EXIT;
 438 }
 439 EXPORT_SYMBOL(lc_watchdog_delete);
 440
 441 /*
 442  * Provided watchdog handlers
 443  */
 444
 445 void lc_watchdog_dumplog(pid_t pid, void *data)
 446 {
 447         libcfs_debug_dumplog_internal((void *)((unsigned long)pid));
 448 }
 449 EXPORT_SYMBOL(lc_watchdog_dumplog);
 450
 451 #else   /* !defined(WITH_WATCHDOG) */
 452
 453 struct lc_watchdog *lc_watchdog_add(int timeout,
 454                                     void (*callback)(pid_t pid, void *),
 455                                     void *data)
 456 {
 457         static struct lc_watchdog      watchdog;
 458         return &watchdog;
 459 }
 460 EXPORT_SYMBOL(lc_watchdog_add);
 461
 462 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 463 {
 464 }
 465 EXPORT_SYMBOL(lc_watchdog_touch);
 466
 467 void lc_watchdog_disable(struct lc_watchdog *lcw)
 468 {
 469 }
 470 EXPORT_SYMBOL(lc_watchdog_disable);
 471
 472 void lc_watchdog_delete(struct lc_watchdog *lcw)
 473 {
 474 }
 475 EXPORT_SYMBOL(lc_watchdog_delete);
 476
 477 #endif