libcfs/libcfs/watchdog.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * libcfs/libcfs/watchdog.c
  37  *
  38  * Author: Jacob Berkman <jacob@clusterfs.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LNET
  42
  43 #include <libcfs/libcfs.h>
  44 #include "tracefile.h"
  45
  46 struct lc_watchdog {
  47         cfs_timer_t           lcw_timer; /* kernel timer */
  48         cfs_list_t            lcw_list;
  49         cfs_time_t            lcw_last_touched;
  50         cfs_task_t           *lcw_task;
  51
  52         void                (*lcw_callback)(pid_t, void *);
  53         void                 *lcw_data;
  54
  55         pid_t                 lcw_pid;
  56
  57         enum {
  58                 LC_WATCHDOG_DISABLED,
  59                 LC_WATCHDOG_ENABLED,
  60                 LC_WATCHDOG_EXPIRED
  61         } lcw_state;
  62 };
  63
  64 #ifdef WITH_WATCHDOG
  65 /*
  66  * The dispatcher will complete lcw_start_completion when it starts,
  67  * and lcw_stop_completion when it exits.
  68  * Wake lcw_event_waitq to signal timer callback dispatches.
  69  */
  70 static cfs_completion_t lcw_start_completion;
  71 static cfs_completion_t  lcw_stop_completion;
  72 static cfs_waitq_t lcw_event_waitq;
  73
  74 /*
  75  * Set this and wake lcw_event_waitq to stop the dispatcher.
  76  */
  77 enum {
  78         LCW_FLAG_STOP = 0
  79 };
  80 static unsigned long lcw_flags = 0;
  81
  82 /*
  83  * Number of outstanding watchdogs.
  84  * When it hits 1, we start the dispatcher.
  85  * When it hits 0, we stop the distpatcher.
  86  */
  87 static __u32         lcw_refcount = 0;
  88 static CFS_DECLARE_MUTEX(lcw_refcount_sem);
  89
  90 /*
  91  * List of timers that have fired that need their callbacks run by the
  92  * dispatcher.
  93  */
  94 /* BH lock! */
  95 static cfs_spinlock_t lcw_pending_timers_lock = CFS_SPIN_LOCK_UNLOCKED;
  96 static cfs_list_t lcw_pending_timers = \
  97         CFS_LIST_HEAD_INIT(lcw_pending_timers);
  98
  99 /* Last time a watchdog expired */
 100 static cfs_time_t lcw_last_watchdog_time;
 101 static int lcw_recent_watchdog_count;
 102 static cfs_spinlock_t lcw_last_watchdog_lock = CFS_SPIN_LOCK_UNLOCKED;
 103
 104 static void
 105 lcw_dump(struct lc_watchdog *lcw)
 106 {
 107         ENTRY;
 108 #if defined(HAVE_TASKLIST_LOCK)
 109         cfs_read_lock(&tasklist_lock);
 110 #elif defined(HAVE_TASK_RCU)
 111         rcu_read_lock();
 112 #else
 113         CERROR("unable to dump stack because of missing export\n");
 114         RETURN_EXIT;
 115 #endif
 116        if (lcw->lcw_task == NULL) {
 117                 LCONSOLE_WARN("Process " LPPID " was not found in the task "
 118                               "list; watchdog callback may be incomplete\n",
 119                               (int)lcw->lcw_pid);
 120         } else {
 121                 libcfs_debug_dumpstack(lcw->lcw_task);
 122         }
 123
 124 #if defined(HAVE_TASKLIST_LOCK)
 125         cfs_read_unlock(&tasklist_lock);
 126 #elif defined(HAVE_TASK_RCU)
 127         rcu_read_unlock();
 128 #endif
 129         EXIT;
 130 }
 131
 132 static void lcw_cb(ulong_ptr_t data)
 133 {
 134         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 135         cfs_time_t current_time;
 136         cfs_duration_t delta_time;
 137         struct timeval timediff;
 138
 139         ENTRY;
 140
 141         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 142                 EXIT;
 143                 return;
 144         }
 145
 146         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 147         current_time = cfs_time_current();
 148
 149         delta_time = cfs_time_sub(current_time, lcw->lcw_last_touched);
 150         cfs_duration_usec(delta_time, &timediff);
 151
 152         /* Check to see if we should throttle the watchdog timer to avoid
 153          * too many dumps going to the console thus triggering an NMI.
 154          * Normally we would not hold the spin lock over the CWARN but in
 155          * this case we hold it to ensure non ratelimited lcw_dumps are not
 156          * interleaved on the console making them hard to read. */
 157         cfs_spin_lock_bh(&lcw_last_watchdog_lock);
 158         delta_time = cfs_duration_sec(cfs_time_sub(current_time,
 159                                                    lcw_last_watchdog_time));
 160
 161         if (delta_time < libcfs_watchdog_ratelimit &&
 162             lcw_recent_watchdog_count > 3) {
 163                 LCONSOLE_WARN("Service thread pid %u was inactive for "
 164                               "%lu.%.02lus. Watchdog stack traces are limited "
 165                               "to 3 per %d seconds, skipping this one.\n",
 166                               (int)lcw->lcw_pid,
 167                               timediff.tv_sec,
 168                               timediff.tv_usec / 10000,
 169                               libcfs_watchdog_ratelimit);
 170         } else {
 171                 if (delta_time < libcfs_watchdog_ratelimit) {
 172                         lcw_recent_watchdog_count++;
 173                 } else {
 174                         memcpy(&lcw_last_watchdog_time, &current_time,
 175                                sizeof(current_time));
 176                         lcw_recent_watchdog_count = 0;
 177                 }
 178
 179                 /* This warning should appear on the console, but may not get
 180                  * into the logs since we're running in a softirq handler */
 181                 LCONSOLE_WARN("Service thread pid %u was inactive for "
 182                               "%lu.%.02lus. The thread might be hung, or it "
 183                               "might only be slow and will resume later. "
 184                               "Dumping the stack trace for debugging purposes:"
 185                               "\n",
 186                               (int)lcw->lcw_pid,
 187                               timediff.tv_sec,
 188                               timediff.tv_usec / 10000);
 189                 lcw_dump(lcw);
 190         }
 191
 192         cfs_spin_unlock_bh(&lcw_last_watchdog_lock);
 193         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 194
 195         if (cfs_list_empty(&lcw->lcw_list)) {
 196                 cfs_list_add(&lcw->lcw_list, &lcw_pending_timers);
 197                 cfs_waitq_signal(&lcw_event_waitq);
 198         }
 199
 200         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 201
 202         EXIT;
 203 }
 204
 205 static int is_watchdog_fired(void)
 206 {
 207         int rc;
 208
 209         if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags))
 210                 return 1;
 211
 212         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 213         rc = !cfs_list_empty(&lcw_pending_timers);
 214         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 215         return rc;
 216 }
 217
 218 static int lcw_dispatch_main(void *data)
 219 {
 220         int                 rc = 0;
 221         unsigned long       flags;
 222         struct lc_watchdog *lcw;
 223
 224         ENTRY;
 225
 226         cfs_daemonize("lc_watchdogd");
 227
 228         SIGNAL_MASK_LOCK(current, flags);
 229         sigfillset(&current->blocked);
 230         RECALC_SIGPENDING;
 231         SIGNAL_MASK_UNLOCK(current, flags);
 232
 233         cfs_complete(&lcw_start_completion);
 234
 235         while (1) {
 236                 cfs_wait_event_interruptible(lcw_event_waitq,
 237                                              is_watchdog_fired(), rc);
 238                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 239                 if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 240                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 241
 242                         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 243                         rc = !cfs_list_empty(&lcw_pending_timers);
 244                         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 245                         if (rc) {
 246                                 CERROR("pending timers list was not empty at "
 247                                        "time of watchdog dispatch shutdown\n");
 248                         }
 249                         break;
 250                 }
 251
 252                 cfs_spin_lock_bh(&lcw_pending_timers_lock);
 253                 while (!cfs_list_empty(&lcw_pending_timers)) {
 254
 255                         lcw = cfs_list_entry(lcw_pending_timers.next,
 256                                          struct lc_watchdog,
 257                                          lcw_list);
 258                         cfs_list_del_init(&lcw->lcw_list);
 259                         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 260
 261                         CDEBUG(D_INFO, "found lcw for pid " LPPID "\n", lcw->lcw_pid);
 262
 263                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
 264                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 265
 266                         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 267                 }
 268                 cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 269         }
 270
 271         cfs_complete(&lcw_stop_completion);
 272
 273         RETURN(rc);
 274 }
 275
 276 static void lcw_dispatch_start(void)
 277 {
 278         int rc;
 279
 280         ENTRY;
 281         LASSERT(lcw_refcount == 1);
 282
 283         cfs_init_completion(&lcw_stop_completion);
 284         cfs_init_completion(&lcw_start_completion);
 285         cfs_waitq_init(&lcw_event_waitq);
 286
 287         CDEBUG(D_INFO, "starting dispatch thread\n");
 288         rc = cfs_kernel_thread(lcw_dispatch_main, NULL, 0);
 289         if (rc < 0) {
 290                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 291                 EXIT;
 292                 return;
 293         }
 294         cfs_wait_for_completion(&lcw_start_completion);
 295         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 296
 297         EXIT;
 298 }
 299
 300 static void lcw_dispatch_stop(void)
 301 {
 302         ENTRY;
 303         LASSERT(lcw_refcount == 0);
 304
 305         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 306
 307         cfs_set_bit(LCW_FLAG_STOP, &lcw_flags);
 308         cfs_waitq_signal(&lcw_event_waitq);
 309
 310         cfs_wait_for_completion(&lcw_stop_completion);
 311
 312         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 313
 314         EXIT;
 315 }
 316
 317 struct lc_watchdog *lc_watchdog_add(int timeout,
 318                                     void (*callback)(pid_t, void *),
 319                                     void *data)
 320 {
 321         struct lc_watchdog *lcw = NULL;
 322         ENTRY;
 323
 324         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 325         if (lcw == NULL) {
 326                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 327                 RETURN(ERR_PTR(-ENOMEM));
 328         }
 329
 330         lcw->lcw_task     = cfs_current();
 331         lcw->lcw_pid      = cfs_curproc_pid();
 332         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 333         lcw->lcw_data     = data;
 334         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 335
 336         CFS_INIT_LIST_HEAD(&lcw->lcw_list);
 337         cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw);
 338
 339         cfs_down(&lcw_refcount_sem);
 340         if (++lcw_refcount == 1)
 341                 lcw_dispatch_start();
 342         cfs_up(&lcw_refcount_sem);
 343
 344         /* Keep this working in case we enable them by default */
 345         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 346                 lcw->lcw_last_touched = cfs_time_current();
 347                 cfs_timer_arm(&lcw->lcw_timer, cfs_time_seconds(timeout) +
 348                               cfs_time_current());
 349         }
 350
 351         RETURN(lcw);
 352 }
 353 EXPORT_SYMBOL(lc_watchdog_add);
 354
 355 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 356 {
 357         cfs_time_t newtime = cfs_time_current();;
 358
 359         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 360                 struct timeval timediff;
 361                 cfs_time_t delta_time = cfs_time_sub(newtime,
 362                                                      lcw->lcw_last_touched);
 363                 cfs_duration_usec(delta_time, &timediff);
 364
 365                 LCONSOLE_WARN("Service thread pid %u %s after %lu.%.02lus. "
 366                               "This indicates the system was overloaded (too "
 367                               "many service threads, or there were not enough "
 368                               "hardware resources).\n",
 369                               lcw->lcw_pid,
 370                               message,
 371                               timediff.tv_sec,
 372                               timediff.tv_usec / 10000);
 373         }
 374         lcw->lcw_last_touched = newtime;
 375 }
 376
 377 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 378 {
 379         ENTRY;
 380         LASSERT(lcw != NULL);
 381
 382         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 383         cfs_list_del_init(&lcw->lcw_list);
 384         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 385
 386         lcw_update_time(lcw, "resumed");
 387         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 388
 389         cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() +
 390                       cfs_time_seconds(timeout));
 391
 392         EXIT;
 393 }
 394 EXPORT_SYMBOL(lc_watchdog_touch);
 395
 396 void lc_watchdog_disable(struct lc_watchdog *lcw)
 397 {
 398         ENTRY;
 399         LASSERT(lcw != NULL);
 400
 401         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 402         if (!cfs_list_empty(&lcw->lcw_list))
 403                 cfs_list_del_init(&lcw->lcw_list);
 404         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 405
 406         lcw_update_time(lcw, "completed");
 407         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 408
 409         EXIT;
 410 }
 411 EXPORT_SYMBOL(lc_watchdog_disable);
 412
 413 void lc_watchdog_delete(struct lc_watchdog *lcw)
 414 {
 415         ENTRY;
 416         LASSERT(lcw != NULL);
 417
 418         cfs_timer_disarm(&lcw->lcw_timer);
 419
 420         lcw_update_time(lcw, "stopped");
 421
 422         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 423         if (!cfs_list_empty(&lcw->lcw_list))
 424                 cfs_list_del_init(&lcw->lcw_list);
 425         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 426
 427         cfs_down(&lcw_refcount_sem);
 428         if (--lcw_refcount == 0)
 429                 lcw_dispatch_stop();
 430         cfs_up(&lcw_refcount_sem);
 431
 432         LIBCFS_FREE(lcw, sizeof(*lcw));
 433
 434         EXIT;
 435 }
 436 EXPORT_SYMBOL(lc_watchdog_delete);
 437
 438 /*
 439  * Provided watchdog handlers
 440  */
 441
 442 void lc_watchdog_dumplog(pid_t pid, void *data)
 443 {
 444         libcfs_debug_dumplog_internal((void *)((long_ptr_t)pid));
 445 }
 446 EXPORT_SYMBOL(lc_watchdog_dumplog);
 447
 448 #else   /* !defined(WITH_WATCHDOG) */
 449
 450 struct lc_watchdog *lc_watchdog_add(int timeout,
 451                                     void (*callback)(pid_t pid, void *),
 452                                     void *data)
 453 {
 454         static struct lc_watchdog      watchdog;
 455         return &watchdog;
 456 }
 457 EXPORT_SYMBOL(lc_watchdog_add);
 458
 459 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 460 {
 461 }
 462 EXPORT_SYMBOL(lc_watchdog_touch);
 463
 464 void lc_watchdog_disable(struct lc_watchdog *lcw)
 465 {
 466 }
 467 EXPORT_SYMBOL(lc_watchdog_disable);
 468
 469 void lc_watchdog_delete(struct lc_watchdog *lcw)
 470 {
 471 }
 472 EXPORT_SYMBOL(lc_watchdog_delete);
 473
 474 #endif