libcfs/libcfs/watchdog.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * libcfs/libcfs/watchdog.c
  37  *
  38  * Author: Jacob Berkman <jacob@clusterfs.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LNET
  42
  43 #include <libcfs/libcfs.h>
  44 #include "tracefile.h"
  45
  46 struct lc_watchdog {
  47         cfs_timer_t       lcw_timer; /* kernel timer */
  48         struct list_head  lcw_list;
  49         struct timeval    lcw_last_touched;
  50         cfs_task_t       *lcw_task;
  51
  52         void            (*lcw_callback)(pid_t, void *);
  53         void             *lcw_data;
  54
  55         pid_t             lcw_pid;
  56         cfs_duration_t    lcw_time; /* time until watchdog fires, jiffies */
  57
  58         enum {
  59                 LC_WATCHDOG_DISABLED,
  60                 LC_WATCHDOG_ENABLED,
  61                 LC_WATCHDOG_EXPIRED
  62         } lcw_state;
  63 };
  64
  65 #ifdef WITH_WATCHDOG
  66 /*
  67  * The dispatcher will complete lcw_start_completion when it starts,
  68  * and lcw_stop_completion when it exits.
  69  * Wake lcw_event_waitq to signal timer callback dispatches.
  70  */
  71 static struct completion lcw_start_completion;
  72 static struct completion lcw_stop_completion;
  73 static cfs_waitq_t lcw_event_waitq;
  74
  75 /*
  76  * Set this and wake lcw_event_waitq to stop the dispatcher.
  77  */
  78 enum {
  79         LCW_FLAG_STOP = 0
  80 };
  81 static unsigned long lcw_flags = 0;
  82
  83 /*
  84  * Number of outstanding watchdogs.
  85  * When it hits 1, we start the dispatcher.
  86  * When it hits 0, we stop the distpatcher.
  87  */
  88 static __u32         lcw_refcount = 0;
  89 static DECLARE_MUTEX(lcw_refcount_sem);
  90
  91 /*
  92  * List of timers that have fired that need their callbacks run by the
  93  * dispatcher.
  94  */
  95 static spinlock_t lcw_pending_timers_lock = SPIN_LOCK_UNLOCKED; /* BH lock! */
  96 static struct list_head lcw_pending_timers = \
  97         CFS_LIST_HEAD_INIT(lcw_pending_timers);
  98
  99 #ifdef HAVE_TASKLIST_LOCK
 100 static void
 101 lcw_dump(struct lc_watchdog *lcw)
 102 {
 103         cfs_task_t *tsk;
 104         ENTRY;
 105
 106         read_lock(&tasklist_lock);
 107         tsk = find_task_by_pid(lcw->lcw_pid);
 108
 109         if (tsk == NULL) {
 110                 CWARN("Process " LPPID " was not found in the task list; "
 111                       "watchdog callback may be incomplete\n", lcw->lcw_pid);
 112         } else if (tsk != lcw->lcw_task) {
 113                 CWARN("The current process " LPPID " did not set the watchdog; "
 114                       "watchdog callback may be incomplete\n", lcw->lcw_pid);
 115         } else {
 116                 libcfs_debug_dumpstack(tsk);
 117         }
 118
 119         read_unlock(&tasklist_lock);
 120         EXIT;
 121 }
 122 #else
 123 static void
 124 lcw_dump(struct lc_watchdog *lcw)
 125 {
 126         CERROR("unable to dump stack because of missing export\n");
 127 }
 128 #endif
 129
 130 static void lcw_cb(ulong_ptr_t data)
 131 {
 132         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 133
 134         ENTRY;
 135
 136         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 137                 EXIT;
 138                 return;
 139         }
 140
 141         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 142
 143         /* NB this warning should appear on the console, but may not get into
 144          * the logs since we're running in a softirq handler */
 145
 146         CWARN("Watchdog triggered for pid: " LPPID " it was inactive for %lds\n",
 147               lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time));
 148         lcw_dump(lcw);
 149
 150         spin_lock_bh(&lcw_pending_timers_lock);
 151
 152         if (list_empty(&lcw->lcw_list)) {
 153                 list_add(&lcw->lcw_list, &lcw_pending_timers);
 154                 cfs_waitq_signal(&lcw_event_waitq);
 155         }
 156
 157         spin_unlock_bh(&lcw_pending_timers_lock);
 158
 159         EXIT;
 160 }
 161
 162 static int is_watchdog_fired(void)
 163 {
 164         int rc;
 165
 166         if (test_bit(LCW_FLAG_STOP, &lcw_flags))
 167                 return 1;
 168
 169         spin_lock_bh(&lcw_pending_timers_lock);
 170         rc = !list_empty(&lcw_pending_timers);
 171         spin_unlock_bh(&lcw_pending_timers_lock);
 172         return rc;
 173 }
 174
 175 static int lcw_dispatch_main(void *data)
 176 {
 177         int                 rc = 0;
 178         unsigned long       flags;
 179         struct lc_watchdog *lcw;
 180
 181         ENTRY;
 182
 183         cfs_daemonize("lc_watchdogd");
 184
 185         SIGNAL_MASK_LOCK(current, flags);
 186         sigfillset(&current->blocked);
 187         RECALC_SIGPENDING;
 188         SIGNAL_MASK_UNLOCK(current, flags);
 189
 190         complete(&lcw_start_completion);
 191
 192         while (1) {
 193                 cfs_wait_event_interruptible(lcw_event_waitq, is_watchdog_fired(), rc);
 194                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 195                 if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 196                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
 197
 198                         spin_lock_bh(&lcw_pending_timers_lock);
 199                         rc = !list_empty(&lcw_pending_timers);
 200                         spin_unlock_bh(&lcw_pending_timers_lock);
 201                         if (rc) {
 202                                 CERROR("pending timers list was not empty at "
 203                                        "time of watchdog dispatch shutdown\n");
 204                         }
 205                         break;
 206                 }
 207
 208                 spin_lock_bh(&lcw_pending_timers_lock);
 209                 while (!list_empty(&lcw_pending_timers)) {
 210
 211                         lcw = list_entry(lcw_pending_timers.next,
 212                                          struct lc_watchdog,
 213                                          lcw_list);
 214                         list_del_init(&lcw->lcw_list);
 215                         spin_unlock_bh(&lcw_pending_timers_lock);
 216
 217                         CDEBUG(D_INFO, "found lcw for pid " LPPID ": inactive for "
 218                                "%lds\n", lcw->lcw_pid, cfs_duration_sec(lcw->lcw_time));
 219
 220                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED)
 221                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 222
 223                         spin_lock_bh(&lcw_pending_timers_lock);
 224                 }
 225                 spin_unlock_bh(&lcw_pending_timers_lock);
 226         }
 227
 228         complete(&lcw_stop_completion);
 229
 230         RETURN(rc);
 231 }
 232
 233 static void lcw_dispatch_start(void)
 234 {
 235         int rc;
 236
 237         ENTRY;
 238         LASSERT(lcw_refcount == 1);
 239
 240         init_completion(&lcw_stop_completion);
 241         init_completion(&lcw_start_completion);
 242         cfs_waitq_init(&lcw_event_waitq);
 243
 244         CDEBUG(D_INFO, "starting dispatch thread\n");
 245         rc = kernel_thread(lcw_dispatch_main, NULL, 0);
 246         if (rc < 0) {
 247                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
 248                 EXIT;
 249                 return;
 250         }
 251         wait_for_completion(&lcw_start_completion);
 252         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 253
 254         EXIT;
 255 }
 256
 257 static void lcw_dispatch_stop(void)
 258 {
 259         ENTRY;
 260         LASSERT(lcw_refcount == 0);
 261
 262         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 263
 264         set_bit(LCW_FLAG_STOP, &lcw_flags);
 265         cfs_waitq_signal(&lcw_event_waitq);
 266
 267         wait_for_completion(&lcw_stop_completion);
 268
 269         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 270
 271         EXIT;
 272 }
 273
 274 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 275                                     void (*callback)(pid_t, void *),
 276                                     void *data)
 277 {
 278         struct lc_watchdog *lcw = NULL;
 279         ENTRY;
 280
 281         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 282         if (lcw == NULL) {
 283                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 284                 RETURN(ERR_PTR(-ENOMEM));
 285         }
 286
 287         lcw->lcw_task     = cfs_current();
 288         lcw->lcw_pid      = cfs_curproc_pid();
 289         lcw->lcw_time     = cfs_time_seconds(timeout_ms) / 1000;
 290         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 291         lcw->lcw_data     = data;
 292         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 293
 294         CFS_INIT_LIST_HEAD(&lcw->lcw_list);
 295         cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw);
 296
 297         down(&lcw_refcount_sem);
 298         if (++lcw_refcount == 1)
 299                 lcw_dispatch_start();
 300         up(&lcw_refcount_sem);
 301
 302         /* Keep this working in case we enable them by default */
 303         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 304                 do_gettimeofday(&lcw->lcw_last_touched);
 305                 cfs_timer_arm(&lcw->lcw_timer, lcw->lcw_time +
 306                               cfs_time_current());
 307         }
 308
 309         RETURN(lcw);
 310 }
 311 EXPORT_SYMBOL(lc_watchdog_add);
 312
 313 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 314 {
 315         struct timeval newtime;
 316         struct timeval timediff;
 317
 318         do_gettimeofday(&newtime);
 319         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 320                 cfs_timeval_sub(&newtime, &lcw->lcw_last_touched, &timediff);
 321                 CWARN("Expired watchdog for pid " LPPID " %s after %lu.%.4lus\n",
 322                       lcw->lcw_pid,
 323                       message,
 324                       timediff.tv_sec,
 325                       timediff.tv_usec / 100);
 326         }
 327         lcw->lcw_last_touched = newtime;
 328 }
 329
 330 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 331 {
 332         ENTRY;
 333         LASSERT(lcw != NULL);
 334
 335         spin_lock_bh(&lcw_pending_timers_lock);
 336         list_del_init(&lcw->lcw_list);
 337         spin_unlock_bh(&lcw_pending_timers_lock);
 338
 339         lcw_update_time(lcw, "touched");
 340         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 341
 342         cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() +
 343                       cfs_time_seconds(timeout_ms) / 1000);
 344
 345         EXIT;
 346 }
 347 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 348
 349 /* deprecated - use above instead */
 350 void lc_watchdog_touch(struct lc_watchdog *lcw)
 351 {
 352         lc_watchdog_touch_ms(lcw, (int)cfs_duration_sec(lcw->lcw_time) * 1000);
 353 }
 354 EXPORT_SYMBOL(lc_watchdog_touch);
 355
 356 void lc_watchdog_disable(struct lc_watchdog *lcw)
 357 {
 358         ENTRY;
 359         LASSERT(lcw != NULL);
 360
 361         spin_lock_bh(&lcw_pending_timers_lock);
 362         if (!list_empty(&lcw->lcw_list))
 363                 list_del_init(&lcw->lcw_list);
 364         spin_unlock_bh(&lcw_pending_timers_lock);
 365
 366         lcw_update_time(lcw, "disabled");
 367         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 368
 369         EXIT;
 370 }
 371 EXPORT_SYMBOL(lc_watchdog_disable);
 372
 373 void lc_watchdog_delete(struct lc_watchdog *lcw)
 374 {
 375         ENTRY;
 376         LASSERT(lcw != NULL);
 377
 378         cfs_timer_disarm(&lcw->lcw_timer);
 379
 380         lcw_update_time(lcw, "deleted");
 381
 382         spin_lock_bh(&lcw_pending_timers_lock);
 383         if (!list_empty(&lcw->lcw_list))
 384                 list_del_init(&lcw->lcw_list);
 385         spin_unlock_bh(&lcw_pending_timers_lock);
 386
 387         down(&lcw_refcount_sem);
 388         if (--lcw_refcount == 0)
 389                 lcw_dispatch_stop();
 390         up(&lcw_refcount_sem);
 391
 392         LIBCFS_FREE(lcw, sizeof(*lcw));
 393
 394         EXIT;
 395 }
 396 EXPORT_SYMBOL(lc_watchdog_delete);
 397
 398 /*
 399  * Provided watchdog handlers
 400  */
 401
 402 void lc_watchdog_dumplog(pid_t pid, void *data)
 403 {
 404         libcfs_debug_dumplog_internal((void *)((long_ptr_t)pid));
 405 }
 406 EXPORT_SYMBOL(lc_watchdog_dumplog);
 407
 408 #else   /* !defined(WITH_WATCHDOG) */
 409
 410 struct lc_watchdog *lc_watchdog_add(int timeout_ms,
 411                                     void (*callback)(pid_t pid, void *),
 412                                     void *data)
 413 {
 414         static struct lc_watchdog      watchdog;
 415         return &watchdog;
 416 }
 417 EXPORT_SYMBOL(lc_watchdog_add);
 418
 419 void lc_watchdog_touch_ms(struct lc_watchdog *lcw, int timeout_ms)
 420 {
 421 }
 422 EXPORT_SYMBOL(lc_watchdog_touch_ms);
 423
 424 void lc_watchdog_touch(struct lc_watchdog *lcw)
 425 {
 426 }
 427 EXPORT_SYMBOL(lc_watchdog_touch);
 428
 429 void lc_watchdog_disable(struct lc_watchdog *lcw)
 430 {
 431 }
 432 EXPORT_SYMBOL(lc_watchdog_disable);
 433
 434 void lc_watchdog_delete(struct lc_watchdog *lcw)
 435 {
 436 }
 437 EXPORT_SYMBOL(lc_watchdog_delete);
 438
 439 #endif