libcfs/libcfs/watchdog.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see [sun.com URL with a
  20  * copy of GPLv2].
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * libcfs/libcfs/watchdog.c
  37  *
  38  * Author: Jacob Berkman <jacob@clusterfs.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LNET
  42
  43 #include <libcfs/libcfs.h>
  44 #include "tracefile.h"
  45
  46 struct lc_watchdog {
  47         cfs_timer_t       lcw_timer; /* kernel timer */
  48         struct list_head  lcw_list;
  49         struct timeval    lcw_last_touched;
  50         cfs_task_t       *lcw_task;
  51
  52         void            (*lcw_callback)(pid_t, void *);
  53         void             *lcw_data;
  54
  55         pid_t             lcw_pid;
  56         cfs_duration_t    lcw_time; /* time until watchdog fires, jiffies */
  57
  58         enum {
  59                 LC_WATCHDOG_DISABLED,
  60                 LC_WATCHDOG_ENABLED,
  61                 LC_WATCHDOG_EXPIRED
  62         } lcw_state;
  63 };
  64
  65 #ifdef WITH_WATCHDOG
  66 /*
  67  * The dispatcher will complete lcw_start_completion when it starts,
  68  * and lcw_stop_completion when it exits.
  69  * Wake lcw_event_waitq to signal timer callback dispatches.
  70  */
  71 static struct completion lcw_start_completion;
  72 static struct completion lcw_stop_completion;
  73 static wait_queue_head_t lcw_event_waitq;
  74
  75 /*
  76  * Set this and wake lcw_event_waitq to stop the dispatcher.
  77  */
  78 enum {
  79         LCW_FLAG_STOP = 0
  80 };
  81 static unsigned long lcw_flags = 0;
  82
  83 /*
  84  * Number of outstanding watchdogs.
  85  * When it hits 1, we start the dispatcher.
  86  * When it hits 0, we stop the distpatcher.
  87  */
  88 static __u32         lcw_refcount = 0;
  89 static DECLARE_MUTEX(lcw_refcount_sem);
  90
  91 /*
  92  * List of timers that have fired that need their callbacks run by the
  93  * dispatcher.
  94  */
  95 static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */
  96 static struct list_head lcw_pending_timers = \
  97         LIST_HEAD_INIT(lcw_pending_timers);
  98
  99 #ifdef HAVE_TASKLIST_LOCK
 100 static void
 101 lcw_dump(struct lc_watchdog *lcw)
 102 {
 103         cfs_task_t *tsk;
 104         ENTRY;
 105
 106         read_lock(&tasklist_lock);
 107         tsk = find_task_by_pid(lcw->lcw_pid);
 108
 109         if (tsk == NULL) {
 110                 CWARN("Process %d was not found in the task list; "
 111                       "watchdog callback may be incomplete\n", (int)lcw->lcw_pid);
 112         } else if (tsk != lcw->lcw_task) {
 113                 CWARN("The current process %d did not set the watchdog; "
 114                       "watchdog callback may be incomplete\n", (int)lcw->lcw_pid);
 115         } else {
 116                 libcfs_debug_dumpstack(tsk);
 117         }
 118
 119         read_unlock(&tasklist_lock);
 120         EXIT;
 121 }
 122 #else
 123 static void
 124 lcw_dump(struct lc_watchdog *lcw)
 125 {
 126         CERROR("unable to dump stack because of missing export\n");
 127 }
 128 #endif
 129
 130 static void lcw_cb(unsigned long data)
 131 {
 132         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 133
 134         ENTRY;
 135
 136         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 137                 EXIT;
 138                 return;
 139         }
 140
 141         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 142
 143         /* NB this warning should appear on the console, but may not get into
 144          * the logs since we're running in a softirq handler */
 145
 146         CWARN("Watchdog triggered for pid %d: it was inactive for %lds\n",
 147               (int)lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time));
 148         lcw_dump(lcw);
 149
 150         spin_lock_bh(&lcw_pending_timers_lock);
 151
 152         if (list_empty(&lcw->lcw_list)) {
 153                 list_add(&lcw->lcw_list, &lcw_pending_timers);
 154                 wake_up(&lcw_event_waitq);
 155         }
 156
 157         spin_unlock_bh(&lcw_pending_timers_lock);
 158
 159         EXIT;
 160 }
 161
 162 static int is_watchdog_fired(void)
 163 {
 164         int rc;
 165
 166         if (test_bit(LCW_FLAG_STOP, &lcw_flags))
 167                 return 1;
 168
 169         spin_lock_bh(&lcw_pending_timers_lock);
 170         rc = !list_empty(&lcw_pending_timers);
 171         spin_unlock_bh(&lcw_pending_timers_lock);
 172         return rc;
 173 }
 174
 175 static int lcw_dispatch_main(void *data)
 176 {
 177         int                 rc = 0;
 178         unsigned long       flags;
 179         struct lc_watchdog *lcw;
 180
 181         ENTRY;
 182
 183         cfs_daemonize("lc_watchdogd");
 184
 185         SIGNAL_MASK_LOCK(current, flags);
 186         sigfillset(&current->blocked);
 187         RECALC_SIGPENDING;
 188         SIGNAL_MASK_UNLOCK(current, flags);
 189
 190         complete(&lcw_start_completion);
 191
 192         while (1) {
 193                 wait_event_interruptible(lcw_event_waitq, is_watchdog_fired());
 194                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 195                 if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 196                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 197
 198                         spin_lock_bh(&lcw_pending_timers_lock);
 199                         rc = !list_empty(&lcw_pending_timers);
 200                         spin_unlock_bh(&lcw_pending_timers_lock);
 201                         if (rc) {
 202                                 CERROR("pending timers list was not empty at "
 203                                        "time of watchdog dispatch shutdown\n");
 204                         }
 205                         break;
 206                 }
 207
 208                 spin_lock_bh(&lcw_pending_timers_lock);
 209                 while (!list_empty(&lcw_pending_timers)) {
 210
 211                         lcw = list_entry(lcw_pending_timers.next,
 212                                          struct lc_watchdog,
 213                                          lcw_list);
 214                         list_del_init(&lcw->lcw_list);
 215                         spin_unlock_bh(&lcw_pending_timers_lock);
 216
 217                         CDEBUG(D_INFO, "found lcw for pid %d: inactive for "
 218                                "%lds\n", (int)lcw->lcw_pid,
 219                                cfs_duration_sec(lcw->lcw_time));
 220
 221                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
 222                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 223
 224                         spin_lock_bh(&lcw_pending_timers_lock);
 225                 }
 226                 spin_unlock_bh(&lcw_pending_timers_lock);
 227         }
 228
 229         complete(&lcw_stop_completion);
 230
 231         RETURN(rc);
 232 }
 233
 234 static void lcw_dispatch_start(void)
 235 {
 236         int rc;
 237
 238         ENTRY;
 239         LASSERT(lcw_refcount == 1);
 240
 241         init_completion(&lcw_stop_completion);
 242         init_completion(&lcw_start_completion);
 243         init_waitqueue_head(&lcw_event_waitq);
 244
 245         CDEBUG(D_INFO, "starting dispatch thread\n");
 246         rc = kernel_thread(lcw_dispatch_main, NULL, 0);
 247         if (rc < 0) {
 248                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 249                 EXIT;
 250                 return;
 251         }
 252         wait_for_completion(&lcw_start_completion);
 253         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 254
 255         EXIT;
 256 }
 257
 258 static void lcw_dispatch_stop(void)
 259 {
 260         ENTRY;
 261         LASSERT(lcw_refcount == 0);
 262
 263         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 264
 265         set_bit(LCW_FLAG_STOP, &lcw_flags);
 266         wake_up(&lcw_event_waitq);
 267
 268         wait_for_completion(&lcw_stop_completion);
 269
 270         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 271
 272         EXIT;
 273 }
 274
 275 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 276                                     void (*callback)(pid_t, void *),
 277                                     void *data)
 278 {
 279         struct lc_watchdog *lcw = NULL;
 280         ENTRY;
 281
 282         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 283         if (lcw == NULL) {
 284                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 285                 RETURN(ERR_PTR(-ENOMEM));
 286         }
 287
 288         lcw->lcw_task     = cfs_current();
 289         lcw->lcw_pid      = cfs_curproc_pid();
 290         lcw->lcw_time     = cfs_time_seconds(timeout_ms) / 1000;
 291         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 292         lcw->lcw_data     = data;
 293         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 294
 295         INIT_LIST_HEAD(&lcw->lcw_list);
 296
 297         lcw->lcw_timer.function = lcw_cb;
 298         lcw->lcw_timer.data = (unsigned long)lcw;
 299         lcw->lcw_timer.expires = jiffies + lcw->lcw_time;
 300         init_timer(&lcw->lcw_timer);
 301
 302         down(&lcw_refcount_sem);
 303         if (++lcw_refcount == 1)
 304                 lcw_dispatch_start();
 305         up(&lcw_refcount_sem);
 306
 307         /* Keep this working in case we enable them by default */
 308         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 309                 do_gettimeofday(&lcw->lcw_last_touched);
 310                 add_timer(&lcw->lcw_timer);
 311         }
 312
 313         RETURN(lcw);
 314 }
 315 EXPORT_SYMBOL(lc_watchdog_add);
 316
 317 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 318 {
 319         struct timeval newtime;
 320         struct timeval timediff;
 321
 322         do_gettimeofday(&newtime);
 323         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 324                 cfs_timeval_sub(&newtime, &lcw->lcw_last_touched, &timediff);
 325                 CWARN("Expired watchdog for pid %d %s after %lu.%.4lus\n",
 326                       lcw->lcw_pid,
 327                       message,
 328                       timediff.tv_sec,
 329                       timediff.tv_usec / 100);
 330         }
 331         lcw->lcw_last_touched = newtime;
 332 }
 333
 334 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 335 {
 336         ENTRY;
 337         LASSERT(lcw != NULL);
 338
 339         spin_lock_bh(&lcw_pending_timers_lock);
 340         list_del_init(&lcw->lcw_list);
 341         spin_unlock_bh(&lcw_pending_timers_lock);
 342
 343         lcw_update_time(lcw, "touched");
 344         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 345
 346         mod_timer(&lcw->lcw_timer, jiffies +
 347                   cfs_time_seconds(timeout_ms) / 1000);
 348
 349         EXIT;
 350 }
 351 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 352
 353 /* deprecated - use above instead */
 354 void lc_watchdog_touch(struct lc_watchdog *lcw)
 355 {
 356         lc_watchdog_touch_ms(lcw, cfs_duration_sec(lcw->lcw_time) * 1000);
 357 }
 358 EXPORT_SYMBOL(lc_watchdog_touch);
 359
 360 void lc_watchdog_disable(struct lc_watchdog *lcw)
 361 {
 362         ENTRY;
 363         LASSERT(lcw != NULL);
 364
 365         spin_lock_bh(&lcw_pending_timers_lock);
 366         if (!list_empty(&lcw->lcw_list))
 367                 list_del_init(&lcw->lcw_list);
 368         spin_unlock_bh(&lcw_pending_timers_lock);
 369
 370         lcw_update_time(lcw, "disabled");
 371         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 372
 373         EXIT;
 374 }
 375 EXPORT_SYMBOL(lc_watchdog_disable);
 376
 377 void lc_watchdog_delete(struct lc_watchdog *lcw)
 378 {
 379         ENTRY;
 380         LASSERT(lcw != NULL);
 381
 382         del_timer(&lcw->lcw_timer);
 383
 384         lcw_update_time(lcw, "deleted");
 385
 386         spin_lock_bh(&lcw_pending_timers_lock);
 387         if (!list_empty(&lcw->lcw_list))
 388                 list_del_init(&lcw->lcw_list);
 389         spin_unlock_bh(&lcw_pending_timers_lock);
 390
 391         down(&lcw_refcount_sem);
 392         if (--lcw_refcount == 0)
 393                 lcw_dispatch_stop();
 394         up(&lcw_refcount_sem);
 395
 396         LIBCFS_FREE(lcw, sizeof(*lcw));
 397
 398         EXIT;
 399 }
 400 EXPORT_SYMBOL(lc_watchdog_delete);
 401
 402 /*
 403  * Provided watchdog handlers
 404  */
 405
 406 void lc_watchdog_dumplog(pid_t pid, void *data)
 407 {
 408         libcfs_debug_dumplog_internal((void *)((unsigned long)pid));
 409 }
 410 EXPORT_SYMBOL(lc_watchdog_dumplog);
 411
 412 #else   /* !defined(WITH_WATCHDOG) */
 413
 414 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 415                                     void (*callback)(pid_t pid, void *),
 416                                     void *data)
 417 {
 418         static struct lc_watchdog      watchdog;
 419         return &watchdog;
 420 }
 421 EXPORT_SYMBOL(lc_watchdog_add);
 422
 423 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 424 {
 425 }
 426 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 427
 428 void lc_watchdog_touch(struct lc_watchdog *lcw)
 429 {
 430 }
 431 EXPORT_SYMBOL(lc_watchdog_touch);
 432
 433 void lc_watchdog_disable(struct lc_watchdog *lcw)
 434 {
 435 }
 436 EXPORT_SYMBOL(lc_watchdog_disable);
 437
 438 void lc_watchdog_delete(struct lc_watchdog *lcw)
 439 {
 440 }
 441 EXPORT_SYMBOL(lc_watchdog_delete);
 442
 443 #endif