libcfs/libcfs/watchdog.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  */
  30 /*
  31  * This file is part of Lustre, http://www.lustre.org/
  32  * Lustre is a trademark of Sun Microsystems, Inc.
  33  *
  34  * libcfs/libcfs/watchdog.c
  35  *
  36  * Author: Jacob Berkman <jacob@clusterfs.com>
  37  */
  38
  39 #define DEBUG_SUBSYSTEM S_LNET
  40
  41 #include <libcfs/libcfs.h>
  42 #include "tracefile.h"
  43
  44 struct lc_watchdog {
  45         cfs_spinlock_t  lcw_lock;     /* check or change lcw_list */
  46         int             lcw_refcount; /* must hold lcw_pending_timers_lock */
  47         cfs_timer_t     lcw_timer;    /* kernel timer */
  48         cfs_list_t      lcw_list;     /* chain on pending list */
  49         cfs_time_t      lcw_last_touched; /* last touched stamp */
  50         cfs_task_t     *lcw_task;     /* owner task */
  51         void          (*lcw_callback)(pid_t, void *);
  52         void           *lcw_data;
  53
  54         pid_t           lcw_pid;
  55
  56         enum {
  57                 LC_WATCHDOG_DISABLED,
  58                 LC_WATCHDOG_ENABLED,
  59                 LC_WATCHDOG_EXPIRED
  60         } lcw_state;
  61 };
  62
  63 #ifdef WITH_WATCHDOG
  64 /*
  65  * The dispatcher will complete lcw_start_completion when it starts,
  66  * and lcw_stop_completion when it exits.
  67  * Wake lcw_event_waitq to signal timer callback dispatches.
  68  */
  69 static cfs_completion_t lcw_start_completion;
  70 static cfs_completion_t  lcw_stop_completion;
  71 static cfs_waitq_t lcw_event_waitq;
  72
  73 /*
  74  * Set this and wake lcw_event_waitq to stop the dispatcher.
  75  */
  76 enum {
  77         LCW_FLAG_STOP = 0
  78 };
  79 static unsigned long lcw_flags = 0;
  80
  81 /*
  82  * Number of outstanding watchdogs.
  83  * When it hits 1, we start the dispatcher.
  84  * When it hits 0, we stop the dispatcher.
  85  */
  86 static __u32         lcw_refcount = 0;
  87 static CFS_DEFINE_MUTEX(lcw_refcount_mutex);
  88
  89 /*
  90  * List of timers that have fired that need their callbacks run by the
  91  * dispatcher.
  92  */
  93 /* BH lock! */
  94 static DEFINE_SPINLOCK(lcw_pending_timers_lock);
  95 static cfs_list_t lcw_pending_timers = CFS_LIST_HEAD_INIT(lcw_pending_timers);
  96
  97 /* Last time a watchdog expired */
  98 static cfs_time_t lcw_last_watchdog_time;
  99 static int lcw_recent_watchdog_count;
 100
 101 static void
 102 lcw_dump(struct lc_watchdog *lcw)
 103 {
 104         ENTRY;
 105 #if defined(HAVE_TASKLIST_LOCK)
 106         cfs_read_lock(&tasklist_lock);
 107 #else
 108         rcu_read_lock();
 109 #endif
 110        if (lcw->lcw_task == NULL) {
 111                 LCONSOLE_WARN("Process " LPPID " was not found in the task "
 112                               "list; watchdog callback may be incomplete\n",
 113                               (int)lcw->lcw_pid);
 114         } else {
 115                 libcfs_debug_dumpstack(lcw->lcw_task);
 116         }
 117
 118 #if defined(HAVE_TASKLIST_LOCK)
 119         cfs_read_unlock(&tasklist_lock);
 120 #else
 121         rcu_read_unlock();
 122 #endif
 123         EXIT;
 124 }
 125
 126 static void lcw_cb(ulong_ptr_t data)
 127 {
 128         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 129         ENTRY;
 130
 131         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 132                 EXIT;
 133                 return;
 134         }
 135
 136         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 137
 138         cfs_spin_lock_bh(&lcw->lcw_lock);
 139         LASSERT(cfs_list_empty(&lcw->lcw_list));
 140
 141         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 142         lcw->lcw_refcount++; /* +1 for pending list */
 143         cfs_list_add(&lcw->lcw_list, &lcw_pending_timers);
 144         cfs_waitq_signal(&lcw_event_waitq);
 145
 146         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 147         cfs_spin_unlock_bh(&lcw->lcw_lock);
 148         EXIT;
 149 }
 150
 151 static int is_watchdog_fired(void)
 152 {
 153         int rc;
 154
 155         if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags))
 156                 return 1;
 157
 158         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 159         rc = !cfs_list_empty(&lcw_pending_timers);
 160         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 161         return rc;
 162 }
 163
 164 static void lcw_dump_stack(struct lc_watchdog *lcw)
 165 {
 166         cfs_time_t      current_time;
 167         cfs_duration_t  delta_time;
 168         struct timeval  timediff;
 169
 170         current_time = cfs_time_current();
 171         delta_time = cfs_time_sub(current_time, lcw->lcw_last_touched);
 172         cfs_duration_usec(delta_time, &timediff);
 173
 174         /*
 175          * Check to see if we should throttle the watchdog timer to avoid
 176          * too many dumps going to the console thus triggering an NMI.
 177          */
 178         delta_time = cfs_duration_sec(cfs_time_sub(current_time,
 179                                                    lcw_last_watchdog_time));
 180
 181         if (delta_time < libcfs_watchdog_ratelimit &&
 182             lcw_recent_watchdog_count > 3) {
 183                 LCONSOLE_WARN("Service thread pid %u was inactive for "
 184                               "%lu.%.02lus. Watchdog stack traces are limited "
 185                               "to 3 per %d seconds, skipping this one.\n",
 186                               (int)lcw->lcw_pid,
 187                               timediff.tv_sec,
 188                               timediff.tv_usec / 10000,
 189                               libcfs_watchdog_ratelimit);
 190         } else {
 191                 if (delta_time < libcfs_watchdog_ratelimit) {
 192                         lcw_recent_watchdog_count++;
 193                 } else {
 194                         memcpy(&lcw_last_watchdog_time, &current_time,
 195                                sizeof(current_time));
 196                         lcw_recent_watchdog_count = 0;
 197                 }
 198
 199                 LCONSOLE_WARN("Service thread pid %u was inactive for "
 200                               "%lu.%.02lus. The thread might be hung, or it "
 201                               "might only be slow and will resume later. "
 202                               "Dumping the stack trace for debugging purposes:"
 203                               "\n",
 204                               (int)lcw->lcw_pid,
 205                               timediff.tv_sec,
 206                               timediff.tv_usec / 10000);
 207                 lcw_dump(lcw);
 208         }
 209 }
 210
 211 static int lcw_dispatch_main(void *data)
 212 {
 213         int                 rc = 0;
 214         unsigned long       flags;
 215         struct lc_watchdog *lcw;
 216         CFS_LIST_HEAD      (zombies);
 217
 218         ENTRY;
 219
 220         cfs_daemonize("lc_watchdogd");
 221
 222         SIGNAL_MASK_LOCK(current, flags);
 223         sigfillset(&current->blocked);
 224         RECALC_SIGPENDING;
 225         SIGNAL_MASK_UNLOCK(current, flags);
 226
 227         cfs_complete(&lcw_start_completion);
 228
 229         while (1) {
 230                 int dumplog = 1;
 231
 232                 cfs_wait_event_interruptible(lcw_event_waitq,
 233                                              is_watchdog_fired(), rc);
 234                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 235                 if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 236                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 237
 238                         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 239                         rc = !cfs_list_empty(&lcw_pending_timers);
 240                         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 241                         if (rc) {
 242                                 CERROR("pending timers list was not empty at "
 243                                        "time of watchdog dispatch shutdown\n");
 244                         }
 245                         break;
 246                 }
 247
 248                 cfs_spin_lock_bh(&lcw_pending_timers_lock);
 249                 while (!cfs_list_empty(&lcw_pending_timers)) {
 250                         int is_dumplog;
 251
 252                         lcw = cfs_list_entry(lcw_pending_timers.next,
 253                                              struct lc_watchdog, lcw_list);
 254                         /* +1 ref for callback to make sure lwc wouldn't be
 255                          * deleted after releasing lcw_pending_timers_lock */
 256                         lcw->lcw_refcount++;
 257                         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 258
 259                         /* lock ordering */
 260                         cfs_spin_lock_bh(&lcw->lcw_lock);
 261                         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 262
 263                         if (cfs_list_empty(&lcw->lcw_list)) {
 264                                 /* already removed from pending list */
 265                                 lcw->lcw_refcount--; /* -1 ref for callback */
 266                                 if (lcw->lcw_refcount == 0)
 267                                         cfs_list_add(&lcw->lcw_list, &zombies);
 268                                 cfs_spin_unlock_bh(&lcw->lcw_lock);
 269                                 /* still hold lcw_pending_timers_lock */
 270                                 continue;
 271                         }
 272
 273                         cfs_list_del_init(&lcw->lcw_list);
 274                         lcw->lcw_refcount--; /* -1 ref for pending list */
 275
 276                         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 277                         cfs_spin_unlock_bh(&lcw->lcw_lock);
 278
 279                         CDEBUG(D_INFO, "found lcw for pid " LPPID "\n",
 280                                lcw->lcw_pid);
 281                         lcw_dump_stack(lcw);
 282
 283                         is_dumplog = lcw->lcw_callback == lc_watchdog_dumplog;
 284                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED &&
 285                             (dumplog || !is_dumplog)) {
 286                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 287                                 if (dumplog && is_dumplog)
 288                                         dumplog = 0;
 289                         }
 290
 291                         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 292                         lcw->lcw_refcount--; /* -1 ref for callback */
 293                         if (lcw->lcw_refcount == 0)
 294                                 cfs_list_add(&lcw->lcw_list, &zombies);
 295                 }
 296                 cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 297
 298                 while (!cfs_list_empty(&zombies)) {
 299                         lcw = cfs_list_entry(lcw_pending_timers.next,
 300                                          struct lc_watchdog, lcw_list);
 301                         cfs_list_del(&lcw->lcw_list);
 302                         LIBCFS_FREE(lcw, sizeof(*lcw));
 303                 }
 304         }
 305
 306         cfs_complete(&lcw_stop_completion);
 307
 308         RETURN(rc);
 309 }
 310
 311 static void lcw_dispatch_start(void)
 312 {
 313         int rc;
 314
 315         ENTRY;
 316         LASSERT(lcw_refcount == 1);
 317
 318         cfs_init_completion(&lcw_stop_completion);
 319         cfs_init_completion(&lcw_start_completion);
 320         cfs_waitq_init(&lcw_event_waitq);
 321
 322         CDEBUG(D_INFO, "starting dispatch thread\n");
 323         rc = cfs_create_thread(lcw_dispatch_main, NULL, 0);
 324         if (rc < 0) {
 325                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 326                 EXIT;
 327                 return;
 328         }
 329         cfs_wait_for_completion(&lcw_start_completion);
 330         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 331
 332         EXIT;
 333 }
 334
 335 static void lcw_dispatch_stop(void)
 336 {
 337         ENTRY;
 338         LASSERT(lcw_refcount == 0);
 339
 340         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 341
 342         cfs_set_bit(LCW_FLAG_STOP, &lcw_flags);
 343         cfs_waitq_signal(&lcw_event_waitq);
 344
 345         cfs_wait_for_completion(&lcw_stop_completion);
 346
 347         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 348
 349         EXIT;
 350 }
 351
 352 struct lc_watchdog *lc_watchdog_add(int timeout,
 353                                     void (*callback)(pid_t, void *),
 354                                     void *data)
 355 {
 356         struct lc_watchdog *lcw = NULL;
 357         ENTRY;
 358
 359         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 360         if (lcw == NULL) {
 361                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 362                 RETURN(ERR_PTR(-ENOMEM));
 363         }
 364
 365         cfs_spin_lock_init(&lcw->lcw_lock);
 366         lcw->lcw_refcount = 1; /* refcount for owner */
 367         lcw->lcw_task     = cfs_current();
 368         lcw->lcw_pid      = cfs_curproc_pid();
 369         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 370         lcw->lcw_data     = data;
 371         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 372
 373         CFS_INIT_LIST_HEAD(&lcw->lcw_list);
 374         cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw);
 375
 376         cfs_mutex_lock(&lcw_refcount_mutex);
 377         if (++lcw_refcount == 1)
 378                 lcw_dispatch_start();
 379         cfs_mutex_unlock(&lcw_refcount_mutex);
 380
 381         /* Keep this working in case we enable them by default */
 382         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 383                 lcw->lcw_last_touched = cfs_time_current();
 384                 cfs_timer_arm(&lcw->lcw_timer, cfs_time_seconds(timeout) +
 385                               cfs_time_current());
 386         }
 387
 388         RETURN(lcw);
 389 }
 390 EXPORT_SYMBOL(lc_watchdog_add);
 391
 392 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 393 {
 394         cfs_time_t newtime = cfs_time_current();;
 395
 396         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 397                 struct timeval timediff;
 398                 cfs_time_t delta_time = cfs_time_sub(newtime,
 399                                                      lcw->lcw_last_touched);
 400                 cfs_duration_usec(delta_time, &timediff);
 401
 402                 LCONSOLE_WARN("Service thread pid %u %s after %lu.%.02lus. "
 403                               "This indicates the system was overloaded (too "
 404                               "many service threads, or there were not enough "
 405                               "hardware resources).\n",
 406                               lcw->lcw_pid,
 407                               message,
 408                               timediff.tv_sec,
 409                               timediff.tv_usec / 10000);
 410         }
 411         lcw->lcw_last_touched = newtime;
 412 }
 413
 414 static void lc_watchdog_del_pending(struct lc_watchdog *lcw)
 415 {
 416         cfs_spin_lock_bh(&lcw->lcw_lock);
 417         if (unlikely(!cfs_list_empty(&lcw->lcw_list))) {
 418                 cfs_spin_lock_bh(&lcw_pending_timers_lock);
 419                 cfs_list_del_init(&lcw->lcw_list);
 420                 lcw->lcw_refcount--; /* -1 ref for pending list */
 421                 cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 422         }
 423
 424         cfs_spin_unlock_bh(&lcw->lcw_lock);
 425 }
 426
 427 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 428 {
 429         ENTRY;
 430         LASSERT(lcw != NULL);
 431
 432         lc_watchdog_del_pending(lcw);
 433
 434         lcw_update_time(lcw, "resumed");
 435         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 436
 437         cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() +
 438                       cfs_time_seconds(timeout));
 439
 440         EXIT;
 441 }
 442 EXPORT_SYMBOL(lc_watchdog_touch);
 443
 444 void lc_watchdog_disable(struct lc_watchdog *lcw)
 445 {
 446         ENTRY;
 447         LASSERT(lcw != NULL);
 448
 449         lc_watchdog_del_pending(lcw);
 450
 451         lcw_update_time(lcw, "completed");
 452         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 453
 454         EXIT;
 455 }
 456 EXPORT_SYMBOL(lc_watchdog_disable);
 457
 458 void lc_watchdog_delete(struct lc_watchdog *lcw)
 459 {
 460         int dead;
 461
 462         ENTRY;
 463         LASSERT(lcw != NULL);
 464
 465         cfs_timer_disarm(&lcw->lcw_timer);
 466
 467         lcw_update_time(lcw, "stopped");
 468
 469         cfs_spin_lock_bh(&lcw->lcw_lock);
 470         cfs_spin_lock_bh(&lcw_pending_timers_lock);
 471         if (unlikely(!cfs_list_empty(&lcw->lcw_list))) {
 472                 cfs_list_del_init(&lcw->lcw_list);
 473                 lcw->lcw_refcount--; /* -1 ref for pending list */
 474         }
 475
 476         lcw->lcw_refcount--; /* -1 ref for owner */
 477         dead = lcw->lcw_refcount == 0;
 478         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
 479         cfs_spin_unlock_bh(&lcw->lcw_lock);
 480
 481         if (dead)
 482                 LIBCFS_FREE(lcw, sizeof(*lcw));
 483
 484         cfs_mutex_lock(&lcw_refcount_mutex);
 485         if (--lcw_refcount == 0)
 486                 lcw_dispatch_stop();
 487         cfs_mutex_unlock(&lcw_refcount_mutex);
 488
 489         EXIT;
 490 }
 491 EXPORT_SYMBOL(lc_watchdog_delete);
 492
 493 /*
 494  * Provided watchdog handlers
 495  */
 496
 497 void lc_watchdog_dumplog(pid_t pid, void *data)
 498 {
 499         libcfs_debug_dumplog_internal((void *)((long_ptr_t)pid));
 500 }
 501 EXPORT_SYMBOL(lc_watchdog_dumplog);
 502
 503 #else   /* !defined(WITH_WATCHDOG) */
 504
 505 struct lc_watchdog *lc_watchdog_add(int timeout,
 506                                     void (*callback)(pid_t pid, void *),
 507                                     void *data)
 508 {
 509         static struct lc_watchdog      watchdog;
 510         return &watchdog;
 511 }
 512 EXPORT_SYMBOL(lc_watchdog_add);
 513
 514 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 515 {
 516 }
 517 EXPORT_SYMBOL(lc_watchdog_touch);
 518
 519 void lc_watchdog_disable(struct lc_watchdog *lcw)
 520 {
 521 }
 522 EXPORT_SYMBOL(lc_watchdog_disable);
 523
 524 void lc_watchdog_delete(struct lc_watchdog *lcw)
 525 {
 526 }
 527 EXPORT_SYMBOL(lc_watchdog_delete);
 528
 529 #endif