lnet/libcfs/watchdog.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/libcfs/watchdog.c
  37  *
  38  * Author: Jacob Berkman <jacob@clusterfs.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LNET
  42
  43 #include <libcfs/kp30.h>
  44 #include <libcfs/libcfs.h>
  45 #include "tracefile.h"
  46
  47 struct lc_watchdog {
  48         cfs_timer_t       lcw_timer; /* kernel timer */
  49         struct list_head  lcw_list;
  50         struct timeval    lcw_last_touched;
  51         cfs_task_t       *lcw_task;
  52
  53         void            (*lcw_callback)(pid_t, void *);
  54         void             *lcw_data;
  55
  56         pid_t             lcw_pid;
  57         cfs_duration_t    lcw_time; /* time until watchdog fires, jiffies */
  58
  59         enum {
  60                 LC_WATCHDOG_DISABLED,
  61                 LC_WATCHDOG_ENABLED,
  62                 LC_WATCHDOG_EXPIRED
  63         } lcw_state;
  64 };
  65
  66 #ifdef WITH_WATCHDOG
  67 /*
  68  * The dispatcher will complete lcw_start_completion when it starts,
  69  * and lcw_stop_completion when it exits.
  70  * Wake lcw_event_waitq to signal timer callback dispatches.
  71  */
  72 static struct completion lcw_start_completion;
  73 static struct completion lcw_stop_completion;
  74 static wait_queue_head_t lcw_event_waitq;
  75
  76 /*
  77  * Set this and wake lcw_event_waitq to stop the dispatcher.
  78  */
  79 enum {
  80         LCW_FLAG_STOP = 0
  81 };
  82 static unsigned long lcw_flags = 0;
  83
  84 /*
  85  * Number of outstanding watchdogs.
  86  * When it hits 1, we start the dispatcher.
  87  * When it hits 0, we stop the distpatcher.
  88  */
  89 static __u32         lcw_refcount = 0;
  90 static DECLARE_MUTEX(lcw_refcount_sem);
  91
  92 /*
  93  * List of timers that have fired that need their callbacks run by the
  94  * dispatcher.
  95  */
  96 static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */
  97 static struct list_head lcw_pending_timers = \
  98         LIST_HEAD_INIT(lcw_pending_timers);
  99
 100 #ifdef HAVE_TASKLIST_LOCK
 101 static void
 102 lcw_dump(struct lc_watchdog *lcw)
 103 {
 104         cfs_task_t *tsk;
 105         ENTRY;
 106
 107         read_lock(&tasklist_lock);
 108         tsk = find_task_by_pid(lcw->lcw_pid);
 109
 110         if (tsk == NULL) {
 111                 CWARN("Process %d was not found in the task list; "
 112                       "watchdog callback may be incomplete\n", (int)lcw->lcw_pid);
 113         } else if (tsk != lcw->lcw_task) {
 114                 CWARN("The current process %d did not set the watchdog; "
 115                       "watchdog callback may be incomplete\n", (int)lcw->lcw_pid);
 116         } else {
 117                 libcfs_debug_dumpstack(tsk);
 118         }
 119
 120         read_unlock(&tasklist_lock);
 121         EXIT;
 122 }
 123 #else
 124 static void
 125 lcw_dump(struct lc_watchdog *lcw)
 126 {
 127         CERROR("unable to dump stack because of missing export\n");
 128 }
 129 #endif
 130
 131 static void lcw_cb(unsigned long data)
 132 {
 133         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 134
 135         ENTRY;
 136
 137         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 138                 EXIT;
 139                 return;
 140         }
 141
 142         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 143
 144         /* NB this warning should appear on the console, but may not get into
 145          * the logs since we're running in a softirq handler */
 146
 147         CWARN("Watchdog triggered for pid %d: it was inactive for %lds\n",
 148               (int)lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time));
 149         lcw_dump(lcw);
 150
 151         spin_lock_bh(&lcw_pending_timers_lock);
 152
 153         if (list_empty(&lcw->lcw_list)) {
 154                 list_add(&lcw->lcw_list, &lcw_pending_timers);
 155                 wake_up(&lcw_event_waitq);
 156         }
 157
 158         spin_unlock_bh(&lcw_pending_timers_lock);
 159
 160         EXIT;
 161 }
 162
 163 static int is_watchdog_fired(void)
 164 {
 165         int rc;
 166
 167         if (test_bit(LCW_FLAG_STOP, &lcw_flags))
 168                 return 1;
 169
 170         spin_lock_bh(&lcw_pending_timers_lock);
 171         rc = !list_empty(&lcw_pending_timers);
 172         spin_unlock_bh(&lcw_pending_timers_lock);
 173         return rc;
 174 }
 175
 176 static int lcw_dispatch_main(void *data)
 177 {
 178         int                 rc = 0;
 179         unsigned long       flags;
 180         struct lc_watchdog *lcw;
 181
 182         ENTRY;
 183
 184         cfs_daemonize("lc_watchdogd");
 185
 186         SIGNAL_MASK_LOCK(current, flags);
 187         sigfillset(&current->blocked);
 188         RECALC_SIGPENDING;
 189         SIGNAL_MASK_UNLOCK(current, flags);
 190
 191         complete(&lcw_start_completion);
 192
 193         while (1) {
 194                 wait_event_interruptible(lcw_event_waitq, is_watchdog_fired());
 195                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 196                 if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 197                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 198
 199                         spin_lock_bh(&lcw_pending_timers_lock);
 200                         rc = !list_empty(&lcw_pending_timers);
 201                         spin_unlock_bh(&lcw_pending_timers_lock);
 202                         if (rc) {
 203                                 CERROR("pending timers list was not empty at "
 204                                        "time of watchdog dispatch shutdown\n");
 205                         }
 206                         break;
 207                 }
 208
 209                 spin_lock_bh(&lcw_pending_timers_lock);
 210                 while (!list_empty(&lcw_pending_timers)) {
 211
 212                         lcw = list_entry(lcw_pending_timers.next,
 213                                          struct lc_watchdog,
 214                                          lcw_list);
 215                         list_del_init(&lcw->lcw_list);
 216                         spin_unlock_bh(&lcw_pending_timers_lock);
 217
 218                         CDEBUG(D_INFO, "found lcw for pid %d: inactive for "
 219                                "%lds\n", (int)lcw->lcw_pid,
 220                                cfs_duration_sec(lcw->lcw_time));
 221
 222                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
 223                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 224
 225                         spin_lock_bh(&lcw_pending_timers_lock);
 226                 }
 227                 spin_unlock_bh(&lcw_pending_timers_lock);
 228         }
 229
 230         complete(&lcw_stop_completion);
 231
 232         RETURN(rc);
 233 }
 234
 235 static void lcw_dispatch_start(void)
 236 {
 237         int rc;
 238
 239         ENTRY;
 240         LASSERT(lcw_refcount == 1);
 241
 242         init_completion(&lcw_stop_completion);
 243         init_completion(&lcw_start_completion);
 244         init_waitqueue_head(&lcw_event_waitq);
 245
 246         CDEBUG(D_INFO, "starting dispatch thread\n");
 247         rc = kernel_thread(lcw_dispatch_main, NULL, 0);
 248         if (rc < 0) {
 249                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 250                 EXIT;
 251                 return;
 252         }
 253         wait_for_completion(&lcw_start_completion);
 254         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 255
 256         EXIT;
 257 }
 258
 259 static void lcw_dispatch_stop(void)
 260 {
 261         ENTRY;
 262         LASSERT(lcw_refcount == 0);
 263
 264         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 265
 266         set_bit(LCW_FLAG_STOP, &lcw_flags);
 267         wake_up(&lcw_event_waitq);
 268
 269         wait_for_completion(&lcw_stop_completion);
 270
 271         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 272
 273         EXIT;
 274 }
 275
 276 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 277                                     void (*callback)(pid_t, void *),
 278                                     void *data)
 279 {
 280         struct lc_watchdog *lcw = NULL;
 281         ENTRY;
 282
 283         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 284         if (lcw == NULL) {
 285                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 286                 RETURN(ERR_PTR(-ENOMEM));
 287         }
 288
 289         lcw->lcw_task     = cfs_current();
 290         lcw->lcw_pid      = cfs_curproc_pid();
 291         lcw->lcw_time     = cfs_time_seconds(timeout_ms) / 1000;
 292         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 293         lcw->lcw_data     = data;
 294         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 295
 296         INIT_LIST_HEAD(&lcw->lcw_list);
 297
 298         lcw->lcw_timer.function = lcw_cb;
 299         lcw->lcw_timer.data = (unsigned long)lcw;
 300         lcw->lcw_timer.expires = jiffies + lcw->lcw_time;
 301         init_timer(&lcw->lcw_timer);
 302
 303         down(&lcw_refcount_sem);
 304         if (++lcw_refcount == 1)
 305                 lcw_dispatch_start();
 306         up(&lcw_refcount_sem);
 307
 308         /* Keep this working in case we enable them by default */
 309         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 310                 do_gettimeofday(&lcw->lcw_last_touched);
 311                 add_timer(&lcw->lcw_timer);
 312         }
 313
 314         RETURN(lcw);
 315 }
 316 EXPORT_SYMBOL(lc_watchdog_add);
 317
 318 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 319 {
 320         struct timeval newtime;
 321         struct timeval timediff;
 322
 323         do_gettimeofday(&newtime);
 324         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 325                 cfs_timeval_sub(&newtime, &lcw->lcw_last_touched, &timediff);
 326                 CWARN("Expired watchdog for pid %d %s after %lu.%.4lus\n",
 327                       lcw->lcw_pid,
 328                       message,
 329                       timediff.tv_sec,
 330                       timediff.tv_usec / 100);
 331         }
 332         lcw->lcw_last_touched = newtime;
 333 }
 334
 335 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 336 {
 337         ENTRY;
 338         LASSERT(lcw != NULL);
 339
 340         spin_lock_bh(&lcw_pending_timers_lock);
 341         list_del_init(&lcw->lcw_list);
 342         spin_unlock_bh(&lcw_pending_timers_lock);
 343
 344         lcw_update_time(lcw, "touched");
 345         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 346
 347         mod_timer(&lcw->lcw_timer, jiffies +
 348                   cfs_time_seconds(timeout_ms) / 1000);
 349
 350         EXIT;
 351 }
 352 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 353
 354 /* deprecated - use above instead */
 355 void lc_watchdog_touch(struct lc_watchdog *lcw)
 356 {
 357         lc_watchdog_touch_ms(lcw, cfs_duration_sec(lcw->lcw_time) * 1000);
 358 }
 359 EXPORT_SYMBOL(lc_watchdog_touch);
 360
 361 void lc_watchdog_disable(struct lc_watchdog *lcw)
 362 {
 363         ENTRY;
 364         LASSERT(lcw != NULL);
 365
 366         spin_lock_bh(&lcw_pending_timers_lock);
 367         if (!list_empty(&lcw->lcw_list))
 368                 list_del_init(&lcw->lcw_list);
 369         spin_unlock_bh(&lcw_pending_timers_lock);
 370
 371         lcw_update_time(lcw, "disabled");
 372         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 373
 374         EXIT;
 375 }
 376 EXPORT_SYMBOL(lc_watchdog_disable);
 377
 378 void lc_watchdog_delete(struct lc_watchdog *lcw)
 379 {
 380         ENTRY;
 381         LASSERT(lcw != NULL);
 382
 383         del_timer(&lcw->lcw_timer);
 384
 385         lcw_update_time(lcw, "deleted");
 386
 387         spin_lock_bh(&lcw_pending_timers_lock);
 388         if (!list_empty(&lcw->lcw_list))
 389                 list_del_init(&lcw->lcw_list);
 390         spin_unlock_bh(&lcw_pending_timers_lock);
 391
 392         down(&lcw_refcount_sem);
 393         if (--lcw_refcount == 0)
 394                 lcw_dispatch_stop();
 395         up(&lcw_refcount_sem);
 396
 397         LIBCFS_FREE(lcw, sizeof(*lcw));
 398
 399         EXIT;
 400 }
 401 EXPORT_SYMBOL(lc_watchdog_delete);
 402
 403 /*
 404  * Provided watchdog handlers
 405  */
 406
 407 void lc_watchdog_dumplog(pid_t pid, void *data)
 408 {
 409         libcfs_debug_dumplog_internal((void *)((unsigned long)pid));
 410 }
 411 EXPORT_SYMBOL(lc_watchdog_dumplog);
 412
 413 #else   /* !defined(WITH_WATCHDOG) */
 414
 415 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 416                                     void (*callback)(pid_t pid, void *),
 417                                     void *data)
 418 {
 419         static struct lc_watchdog      watchdog;
 420         return &watchdog;
 421 }
 422 EXPORT_SYMBOL(lc_watchdog_add);
 423
 424 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 425 {
 426 }
 427 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 428
 429 void lc_watchdog_touch(struct lc_watchdog *lcw)
 430 {
 431 }
 432 EXPORT_SYMBOL(lc_watchdog_touch);
 433
 434 void lc_watchdog_disable(struct lc_watchdog *lcw)
 435 {
 436 }
 437 EXPORT_SYMBOL(lc_watchdog_disable);
 438
 439 void lc_watchdog_delete(struct lc_watchdog *lcw)
 440 {
 441 }
 442 EXPORT_SYMBOL(lc_watchdog_delete);
 443
 444 #endif