libcfs/libcfs/watchdog.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2014, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * libcfs/libcfs/watchdog.c
  33  *
  34  * Author: Jacob Berkman <jacob@clusterfs.com>
  35  */
  36
  37 #define DEBUG_SUBSYSTEM S_LNET
  38
  39 #include <linux/kthread.h>
  40 #include <libcfs/libcfs.h>
  41 #include "tracefile.h"
  42
  43 struct lc_watchdog {
  44         spinlock_t              lcw_lock;       /* check or change lcw_list */
  45         int                     lcw_refcount;   /* must hold lcw_pending_timers_lock */
  46         struct timer_list       lcw_timer;      /* kernel timer */
  47         struct list_head        lcw_list;       /* chain on pending list */
  48         ktime_t                 lcw_last_touched;/* last touched stamp */
  49         struct task_struct     *lcw_task;       /* owner task */
  50         void                    (*lcw_callback)(pid_t, void *);
  51         void                    *lcw_data;
  52
  53         pid_t                   lcw_pid;
  54
  55         enum {
  56                 LC_WATCHDOG_DISABLED,
  57                 LC_WATCHDOG_ENABLED,
  58                 LC_WATCHDOG_EXPIRED
  59         } lcw_state;
  60 };
  61
  62 #ifdef WITH_WATCHDOG
  63 /*
  64  * The dispatcher will complete lcw_start_completion when it starts,
  65  * and lcw_stop_completion when it exits.
  66  * Wake lcw_event_waitq to signal timer callback dispatches.
  67  */
  68 static struct completion lcw_start_completion;
  69 static struct completion  lcw_stop_completion;
  70 static wait_queue_head_t lcw_event_waitq;
  71
  72 /*
  73  * Set this and wake lcw_event_waitq to stop the dispatcher.
  74  */
  75 enum {
  76         LCW_FLAG_STOP = 0
  77 };
  78 static unsigned long lcw_flags = 0;
  79
  80 /*
  81  * Number of outstanding watchdogs.
  82  * When it hits 1, we start the dispatcher.
  83  * When it hits 0, we stop the dispatcher.
  84  */
  85 static __u32         lcw_refcount = 0;
  86 static DEFINE_MUTEX(lcw_refcount_mutex);
  87
  88 /*
  89  * List of timers that have fired that need their callbacks run by the
  90  * dispatcher.
  91  */
  92 /* BH lock! */
  93 static DEFINE_SPINLOCK(lcw_pending_timers_lock);
  94 static struct list_head lcw_pending_timers = LIST_HEAD_INIT(lcw_pending_timers);
  95
  96 /* Last time a watchdog expired */
  97 static time64_t lcw_last_watchdog_time;
  98 static int lcw_recent_watchdog_count;
  99
 100 static void
 101 lcw_dump(struct lc_watchdog *lcw)
 102 {
 103         ENTRY;
 104         rcu_read_lock();
 105        if (lcw->lcw_task == NULL) {
 106                 LCONSOLE_WARN("Process %d was not found in the task "
 107                               "list; watchdog callback may be incomplete\n",
 108                               (int)lcw->lcw_pid);
 109         } else {
 110                 libcfs_debug_dumpstack(lcw->lcw_task);
 111         }
 112
 113         rcu_read_unlock();
 114         EXIT;
 115 }
 116
 117 static void lcw_cb(uintptr_t data)
 118 {
 119         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
 120         ENTRY;
 121
 122         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
 123                 EXIT;
 124                 return;
 125         }
 126
 127         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
 128
 129         spin_lock_bh(&lcw->lcw_lock);
 130         LASSERT(list_empty(&lcw->lcw_list));
 131
 132         spin_lock_bh(&lcw_pending_timers_lock);
 133         lcw->lcw_refcount++; /* +1 for pending list */
 134         list_add(&lcw->lcw_list, &lcw_pending_timers);
 135         wake_up(&lcw_event_waitq);
 136
 137         spin_unlock_bh(&lcw_pending_timers_lock);
 138         spin_unlock_bh(&lcw->lcw_lock);
 139         EXIT;
 140 }
 141
 142 static int is_watchdog_fired(void)
 143 {
 144         int rc;
 145
 146         if (test_bit(LCW_FLAG_STOP, &lcw_flags))
 147                 return 1;
 148
 149         spin_lock_bh(&lcw_pending_timers_lock);
 150         rc = !list_empty(&lcw_pending_timers);
 151         spin_unlock_bh(&lcw_pending_timers_lock);
 152         return rc;
 153 }
 154
 155 static void lcw_dump_stack(struct lc_watchdog *lcw)
 156 {
 157         time64_t current_time = ktime_get_seconds();
 158         struct timespec64 timediff;
 159         time64_t delta_time;
 160
 161         timediff = ktime_to_timespec64(ktime_sub(ktime_get(),
 162                                        lcw->lcw_last_touched));
 163         /*
 164          * Check to see if we should throttle the watchdog timer to avoid
 165          * too many dumps going to the console thus triggering an NMI.
 166          */
 167         delta_time = current_time - lcw_last_watchdog_time;
 168         if (delta_time < libcfs_watchdog_ratelimit &&
 169             lcw_recent_watchdog_count > 3) {
 170                 LCONSOLE_WARN("Service thread pid %u was inactive for %lu.%.02lus. Watchdog stack traces are limited to 3 per %d seconds, skipping this one.\n",
 171                               (int)lcw->lcw_pid,
 172                               timediff.tv_sec,
 173                               timediff.tv_nsec / (NSEC_PER_SEC / 100),
 174                               libcfs_watchdog_ratelimit);
 175         } else {
 176                 if (delta_time < libcfs_watchdog_ratelimit) {
 177                         lcw_recent_watchdog_count++;
 178                 } else {
 179                         memcpy(&lcw_last_watchdog_time, &current_time,
 180                                sizeof(current_time));
 181                         lcw_recent_watchdog_count = 0;
 182                 }
 183
 184                 LCONSOLE_WARN("Service thread pid %u was inactive for %lu.%.02lus. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes:\n",
 185                               (int)lcw->lcw_pid,
 186                               timediff.tv_sec,
 187                               timediff.tv_nsec / (NSEC_PER_SEC / 100));
 188                 lcw_dump(lcw);
 189         }
 190 }
 191
 192 /*
 193  * Provided watchdog handlers
 194  */
 195
 196 static void lc_watchdog_dumplog(pid_t pid, void *data)
 197 {
 198         libcfs_debug_dumplog_internal((void *)((uintptr_t)pid));
 199 }
 200
 201 static int lcw_dispatch_main(void *data)
 202 {
 203         int                 rc = 0;
 204         struct lc_watchdog *lcw;
 205         struct list_head zombies = LIST_HEAD_INIT(zombies);
 206
 207         ENTRY;
 208
 209         complete(&lcw_start_completion);
 210
 211         while (1) {
 212                 int dumplog = 1;
 213
 214                 rc = wait_event_interruptible(lcw_event_waitq,
 215                                               is_watchdog_fired());
 216                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
 217                 if (test_bit(LCW_FLAG_STOP, &lcw_flags)) {
 218                         CDEBUG(D_INFO, "LCW_FLAG_STOP set, shutting down...\n");
 219
 220                         spin_lock_bh(&lcw_pending_timers_lock);
 221                         rc = !list_empty(&lcw_pending_timers);
 222                         spin_unlock_bh(&lcw_pending_timers_lock);
 223                         if (rc) {
 224                                 CERROR("pending timers list was not empty at "
 225                                        "time of watchdog dispatch shutdown\n");
 226                         }
 227                         break;
 228                 }
 229
 230                 spin_lock_bh(&lcw_pending_timers_lock);
 231                 while (!list_empty(&lcw_pending_timers)) {
 232                         int is_dumplog;
 233
 234                         lcw = list_entry(lcw_pending_timers.next,
 235                                          struct lc_watchdog, lcw_list);
 236                         /* +1 ref for callback to make sure lwc wouldn't be
 237                          * deleted after releasing lcw_pending_timers_lock */
 238                         lcw->lcw_refcount++;
 239                         spin_unlock_bh(&lcw_pending_timers_lock);
 240
 241                         /* lock ordering */
 242                         spin_lock_bh(&lcw->lcw_lock);
 243                         spin_lock_bh(&lcw_pending_timers_lock);
 244
 245                         if (list_empty(&lcw->lcw_list)) {
 246                                 /* already removed from pending list */
 247                                 lcw->lcw_refcount--; /* -1 ref for callback */
 248                                 if (lcw->lcw_refcount == 0)
 249                                         list_add(&lcw->lcw_list, &zombies);
 250                                 spin_unlock_bh(&lcw->lcw_lock);
 251                                 /* still hold lcw_pending_timers_lock */
 252                                 continue;
 253                         }
 254
 255                         list_del_init(&lcw->lcw_list);
 256                         lcw->lcw_refcount--; /* -1 ref for pending list */
 257
 258                         spin_unlock_bh(&lcw_pending_timers_lock);
 259                         spin_unlock_bh(&lcw->lcw_lock);
 260
 261                         CDEBUG(D_INFO, "found lcw for pid %d\n",
 262                                lcw->lcw_pid);
 263                         lcw_dump_stack(lcw);
 264
 265                         is_dumplog = lcw->lcw_callback == lc_watchdog_dumplog;
 266                         if (lcw->lcw_state != LC_WATCHDOG_DISABLED &&
 267                             (dumplog || !is_dumplog)) {
 268                                 lcw->lcw_callback(lcw->lcw_pid, lcw->lcw_data);
 269                                 if (dumplog && is_dumplog)
 270                                         dumplog = 0;
 271                         }
 272
 273                         spin_lock_bh(&lcw_pending_timers_lock);
 274                         lcw->lcw_refcount--; /* -1 ref for callback */
 275                         if (lcw->lcw_refcount == 0)
 276                                 list_add(&lcw->lcw_list, &zombies);
 277                 }
 278                 spin_unlock_bh(&lcw_pending_timers_lock);
 279
 280                 while (!list_empty(&zombies)) {
 281                         lcw = list_entry(zombies.next,
 282                                              struct lc_watchdog, lcw_list);
 283                         list_del_init(&lcw->lcw_list);
 284                         LIBCFS_FREE(lcw, sizeof(*lcw));
 285                 }
 286         }
 287
 288         complete(&lcw_stop_completion);
 289
 290         RETURN(rc);
 291 }
 292
 293 static void lcw_dispatch_start(void)
 294 {
 295         struct task_struct *task;
 296
 297         ENTRY;
 298         LASSERT(lcw_refcount == 1);
 299
 300         init_completion(&lcw_stop_completion);
 301         init_completion(&lcw_start_completion);
 302         init_waitqueue_head(&lcw_event_waitq);
 303
 304         CDEBUG(D_INFO, "starting dispatch thread\n");
 305         task = kthread_run(lcw_dispatch_main, NULL, "lc_watchdogd");
 306         if (IS_ERR(task)) {
 307                 CERROR("error spawning watchdog dispatch thread: %ld\n",
 308                         PTR_ERR(task));
 309                 EXIT;
 310                 return;
 311         }
 312         wait_for_completion(&lcw_start_completion);
 313         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
 314
 315         EXIT;
 316 }
 317
 318 static void lcw_dispatch_stop(void)
 319 {
 320         ENTRY;
 321         LASSERT(lcw_refcount == 0);
 322
 323         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
 324
 325         set_bit(LCW_FLAG_STOP, &lcw_flags);
 326         wake_up(&lcw_event_waitq);
 327
 328         wait_for_completion(&lcw_stop_completion);
 329
 330         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
 331
 332         EXIT;
 333 }
 334
 335 struct lc_watchdog *lc_watchdog_add(int timeout,
 336                                     void (*callback)(pid_t, void *),
 337                                     void *data)
 338 {
 339         struct lc_watchdog *lcw = NULL;
 340         ENTRY;
 341
 342         LIBCFS_ALLOC(lcw, sizeof(*lcw));
 343         if (lcw == NULL) {
 344                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
 345                 RETURN(ERR_PTR(-ENOMEM));
 346         }
 347
 348         spin_lock_init(&lcw->lcw_lock);
 349         lcw->lcw_refcount = 1; /* refcount for owner */
 350         lcw->lcw_task     = current;
 351         lcw->lcw_pid      = current_pid();
 352         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
 353         lcw->lcw_data     = data;
 354         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
 355
 356         INIT_LIST_HEAD(&lcw->lcw_list);
 357         setup_timer(&lcw->lcw_timer, lcw_cb, (unsigned long)lcw);
 358
 359         mutex_lock(&lcw_refcount_mutex);
 360         if (++lcw_refcount == 1)
 361                 lcw_dispatch_start();
 362         mutex_unlock(&lcw_refcount_mutex);
 363
 364         /* Keep this working in case we enable them by default */
 365         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
 366                 lcw->lcw_last_touched = ktime_get();
 367                 mod_timer(&lcw->lcw_timer, cfs_time_seconds(timeout) +
 368                           jiffies);
 369         }
 370
 371         RETURN(lcw);
 372 }
 373 EXPORT_SYMBOL(lc_watchdog_add);
 374
 375 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
 376 {
 377         ktime_t newtime = ktime_get();
 378
 379         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
 380                 ktime_t lapse = ktime_sub(newtime, lcw->lcw_last_touched);
 381                 struct timespec64 timediff;
 382
 383                 timediff = ktime_to_timespec64(lapse);
 384                 LCONSOLE_WARN("Service thread pid %u %s after %lu.%.02lus. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources).\n",
 385                               lcw->lcw_pid, message,
 386                               timediff.tv_sec,
 387                               timediff.tv_nsec / (NSEC_PER_SEC / 100));
 388         }
 389         lcw->lcw_last_touched = newtime;
 390 }
 391
 392 static void lc_watchdog_del_pending(struct lc_watchdog *lcw)
 393 {
 394         spin_lock_bh(&lcw->lcw_lock);
 395         if (unlikely(!list_empty(&lcw->lcw_list))) {
 396                 spin_lock_bh(&lcw_pending_timers_lock);
 397                 list_del_init(&lcw->lcw_list);
 398                 lcw->lcw_refcount--; /* -1 ref for pending list */
 399                 spin_unlock_bh(&lcw_pending_timers_lock);
 400         }
 401
 402         spin_unlock_bh(&lcw->lcw_lock);
 403 }
 404
 405 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 406 {
 407         ENTRY;
 408         LASSERT(lcw != NULL);
 409
 410         lc_watchdog_del_pending(lcw);
 411
 412         lcw_update_time(lcw, "resumed");
 413
 414         mod_timer(&lcw->lcw_timer, jiffies + cfs_time_seconds(timeout));
 415         lcw->lcw_state = LC_WATCHDOG_ENABLED;
 416
 417         EXIT;
 418 }
 419 EXPORT_SYMBOL(lc_watchdog_touch);
 420
 421 void lc_watchdog_disable(struct lc_watchdog *lcw)
 422 {
 423         ENTRY;
 424         LASSERT(lcw != NULL);
 425
 426         lc_watchdog_del_pending(lcw);
 427
 428         lcw_update_time(lcw, "completed");
 429         lcw->lcw_state = LC_WATCHDOG_DISABLED;
 430
 431         EXIT;
 432 }
 433 EXPORT_SYMBOL(lc_watchdog_disable);
 434
 435 void lc_watchdog_delete(struct lc_watchdog *lcw)
 436 {
 437         int dead;
 438
 439         ENTRY;
 440         LASSERT(lcw != NULL);
 441
 442         del_timer(&lcw->lcw_timer);
 443
 444         lcw_update_time(lcw, "stopped");
 445
 446         spin_lock_bh(&lcw->lcw_lock);
 447         spin_lock_bh(&lcw_pending_timers_lock);
 448         if (unlikely(!list_empty(&lcw->lcw_list))) {
 449                 list_del_init(&lcw->lcw_list);
 450                 lcw->lcw_refcount--; /* -1 ref for pending list */
 451         }
 452
 453         lcw->lcw_refcount--; /* -1 ref for owner */
 454         dead = lcw->lcw_refcount == 0;
 455         spin_unlock_bh(&lcw_pending_timers_lock);
 456         spin_unlock_bh(&lcw->lcw_lock);
 457
 458         if (dead)
 459                 LIBCFS_FREE(lcw, sizeof(*lcw));
 460
 461         mutex_lock(&lcw_refcount_mutex);
 462         if (--lcw_refcount == 0)
 463                 lcw_dispatch_stop();
 464         mutex_unlock(&lcw_refcount_mutex);
 465
 466         EXIT;
 467 }
 468 EXPORT_SYMBOL(lc_watchdog_delete);
 469
 470 #else   /* !defined(WITH_WATCHDOG) */
 471
 472 struct lc_watchdog *lc_watchdog_add(int timeout,
 473                                     void (*callback)(pid_t pid, void *),
 474                                     void *data)
 475 {
 476         static struct lc_watchdog      watchdog;
 477         return &watchdog;
 478 }
 479 EXPORT_SYMBOL(lc_watchdog_add);
 480
 481 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
 482 {
 483 }
 484 EXPORT_SYMBOL(lc_watchdog_touch);
 485
 486 void lc_watchdog_disable(struct lc_watchdog *lcw)
 487 {
 488 }
 489 EXPORT_SYMBOL(lc_watchdog_disable);
 490
 491 void lc_watchdog_delete(struct lc_watchdog *lcw)
 492 {
 493 }
 494 EXPORT_SYMBOL(lc_watchdog_delete);
 495
 496 #endif