Whamcloud - gitweb
b22078 wrong value is used by parse_net
[fs/lustre-release.git] / libcfs / libcfs / watchdog.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * GPL HEADER START
5  *
6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 only,
10  * as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License version 2 for more details (a copy is included
16  * in the LICENSE file that accompanied this code).
17  *
18  * You should have received a copy of the GNU General Public License
19  * version 2 along with this program; If not, see
20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
21  *
22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23  * CA 95054 USA or visit www.sun.com if you need additional information or
24  * have any questions.
25  *
26  * GPL HEADER END
27  */
28 /*
29  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
30  * Use is subject to license terms.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * libcfs/libcfs/watchdog.c
37  *
38  * Author: Jacob Berkman <jacob@clusterfs.com>
39  */
40
41 #define DEBUG_SUBSYSTEM S_LNET
42
43 #include <libcfs/libcfs.h>
44 #include "tracefile.h"
45
46 struct lc_watchdog {
47         cfs_timer_t           lcw_timer; /* kernel timer */
48         cfs_list_t            lcw_list;
49         cfs_time_t            lcw_last_touched;
50         cfs_task_t           *lcw_task;
51         cfs_atomic_t          lcw_refcount;
52
53         void                (*lcw_callback)(pid_t, void *);
54         void                 *lcw_data;
55
56         pid_t                 lcw_pid;
57
58         enum {
59                 LC_WATCHDOG_DISABLED,
60                 LC_WATCHDOG_ENABLED,
61                 LC_WATCHDOG_EXPIRED
62         } lcw_state;
63 };
64
65 #ifdef WITH_WATCHDOG
66 /*
67  * The dispatcher will complete lcw_start_completion when it starts,
68  * and lcw_stop_completion when it exits.
69  * Wake lcw_event_waitq to signal timer callback dispatches.
70  */
71 static cfs_completion_t lcw_start_completion;
72 static cfs_completion_t  lcw_stop_completion;
73 static cfs_waitq_t lcw_event_waitq;
74
75 /*
76  * Set this and wake lcw_event_waitq to stop the dispatcher.
77  */
78 enum {
79         LCW_FLAG_STOP = 0
80 };
81 static unsigned long lcw_flags = 0;
82
83 /*
84  * Number of outstanding watchdogs.
85  * When it hits 1, we start the dispatcher.
86  * When it hits 0, we stop the distpatcher.
87  */
88 static __u32         lcw_refcount = 0;
89 static CFS_DECLARE_MUTEX(lcw_refcount_sem);
90
91 /*
92  * List of timers that have fired that need their callbacks run by the
93  * dispatcher.
94  */
95 /* BH lock! */
96 static cfs_spinlock_t lcw_pending_timers_lock = CFS_SPIN_LOCK_UNLOCKED;
97 static cfs_list_t lcw_pending_timers = \
98         CFS_LIST_HEAD_INIT(lcw_pending_timers);
99
100 /* Last time a watchdog expired */
101 static cfs_time_t lcw_last_watchdog_time;
102 static int lcw_recent_watchdog_count;
103
104 static void
105 lcw_dump(struct lc_watchdog *lcw)
106 {
107         ENTRY;
108 #if defined(HAVE_TASKLIST_LOCK)
109         cfs_read_lock(&tasklist_lock);
110 #elif defined(HAVE_TASK_RCU)
111         rcu_read_lock();
112 #else
113         CERROR("unable to dump stack because of missing export\n"); 
114         RETURN_EXIT;
115 #endif
116        if (lcw->lcw_task == NULL) { 
117                 LCONSOLE_WARN("Process " LPPID " was not found in the task "
118                               "list; watchdog callback may be incomplete\n",
119                               (int)lcw->lcw_pid);
120         } else {
121                 libcfs_debug_dumpstack(lcw->lcw_task);
122         }
123
124 #if defined(HAVE_TASKLIST_LOCK)
125         cfs_read_unlock(&tasklist_lock);
126 #elif defined(HAVE_TASK_RCU)
127         rcu_read_unlock();
128 #endif
129         EXIT;
130 }
131
132 static void lcw_cb(ulong_ptr_t data)
133 {
134         struct lc_watchdog *lcw = (struct lc_watchdog *)data;
135         ENTRY;
136
137         if (lcw->lcw_state != LC_WATCHDOG_ENABLED) {
138                 EXIT;
139                 return;
140         }
141
142         lcw->lcw_state = LC_WATCHDOG_EXPIRED;
143
144         cfs_spin_lock_bh(&lcw_pending_timers_lock);
145         cfs_list_add(&lcw->lcw_list, &lcw_pending_timers);
146         cfs_waitq_signal(&lcw_event_waitq);
147         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
148
149         EXIT;
150 }
151
152 static inline void lcw_get(struct lc_watchdog *lcw)
153 {
154         cfs_atomic_inc(&lcw->lcw_refcount);
155 }
156
157 static inline void lcw_put(struct lc_watchdog *lcw)
158 {
159         if (cfs_atomic_dec_and_test(&lcw->lcw_refcount)) {
160                 LASSERT(cfs_list_empty(&lcw->lcw_list));
161                 LIBCFS_FREE(lcw, sizeof(*lcw));
162         }
163 }
164
165 static int is_watchdog_fired(void)
166 {
167         int rc;
168
169         if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags))
170                 return 1;
171
172         cfs_spin_lock_bh(&lcw_pending_timers_lock);
173         rc = !cfs_list_empty(&lcw_pending_timers);
174         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
175         return rc;
176 }
177
178 static void lcw_dump_stack(struct lc_watchdog *lcw)
179 {
180         cfs_time_t      current_time;
181         cfs_duration_t  delta_time;
182         struct timeval  timediff;
183
184         current_time = cfs_time_current();
185         delta_time = cfs_time_sub(current_time, lcw->lcw_last_touched);
186         cfs_duration_usec(delta_time, &timediff);
187
188         /*
189          * Check to see if we should throttle the watchdog timer to avoid
190          * too many dumps going to the console thus triggering an NMI.
191          */
192         delta_time = cfs_duration_sec(cfs_time_sub(current_time,
193                                                    lcw_last_watchdog_time));
194
195         if (delta_time < libcfs_watchdog_ratelimit &&
196             lcw_recent_watchdog_count > 3) {
197                 LCONSOLE_WARN("Service thread pid %u was inactive for "
198                               "%lu.%.02lus. Watchdog stack traces are limited "
199                               "to 3 per %d seconds, skipping this one.\n",
200                               (int)lcw->lcw_pid,
201                               timediff.tv_sec,
202                               timediff.tv_usec / 10000,
203                               libcfs_watchdog_ratelimit);
204         } else {
205                 if (delta_time < libcfs_watchdog_ratelimit) {
206                         lcw_recent_watchdog_count++;
207                 } else {
208                         memcpy(&lcw_last_watchdog_time, &current_time,
209                                sizeof(current_time));
210                         lcw_recent_watchdog_count = 0;
211                 }
212
213                 LCONSOLE_WARN("Service thread pid %u was inactive for "
214                               "%lu.%.02lus. The thread might be hung, or it "
215                               "might only be slow and will resume later. "
216                               "Dumping the stack trace for debugging purposes:"
217                               "\n",
218                               (int)lcw->lcw_pid,
219                               timediff.tv_sec,
220                               timediff.tv_usec / 10000);
221                 lcw_dump(lcw);
222         }
223 }
224
225 static int lcw_dispatch_main(void *data)
226 {
227         int                 rc = 0;
228         unsigned long       flags;
229         struct lc_watchdog *lcw, *lcwcb;
230
231         ENTRY;
232
233         cfs_daemonize("lc_watchdogd");
234
235         SIGNAL_MASK_LOCK(current, flags);
236         sigfillset(&current->blocked);
237         RECALC_SIGPENDING;
238         SIGNAL_MASK_UNLOCK(current, flags);
239
240         cfs_complete(&lcw_start_completion);
241
242         while (1) {
243                 cfs_wait_event_interruptible(lcw_event_waitq,
244                                              is_watchdog_fired(), rc);
245                 CDEBUG(D_INFO, "Watchdog got woken up...\n");
246                 if (cfs_test_bit(LCW_FLAG_STOP, &lcw_flags)) {
247                         CDEBUG(D_INFO, "LCW_FLAG_STOP was set, shutting down...\n");
248
249                         cfs_spin_lock_bh(&lcw_pending_timers_lock);
250                         rc = !cfs_list_empty(&lcw_pending_timers);
251                         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
252                         if (rc) {
253                                 CERROR("pending timers list was not empty at "
254                                        "time of watchdog dispatch shutdown\n");
255                         }
256                         break;
257                 }
258
259                 lcwcb = NULL;
260                 cfs_spin_lock_bh(&lcw_pending_timers_lock);
261                 while (!cfs_list_empty(&lcw_pending_timers)) {
262
263                         lcw = cfs_list_entry(lcw_pending_timers.next,
264                                          struct lc_watchdog,
265                                          lcw_list);
266                         lcw_get(lcw);
267                         cfs_list_del_init(&lcw->lcw_list);
268                         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
269
270                         CDEBUG(D_INFO, "found lcw for pid " LPPID "\n",
271                                lcw->lcw_pid);
272                         lcw_dump_stack(lcw);
273
274                         if (lcwcb == NULL &&
275                             lcw->lcw_state != LC_WATCHDOG_DISABLED)
276                                 lcwcb = lcw;
277                         else
278                                 lcw_put(lcw);
279                         cfs_spin_lock_bh(&lcw_pending_timers_lock);
280                 }
281                 cfs_spin_unlock_bh(&lcw_pending_timers_lock);
282
283                 /* only do callback once for this batch of lcws */
284                 if (lcwcb != NULL) {
285                         lcwcb->lcw_callback(lcwcb->lcw_pid, lcwcb->lcw_data);
286                         lcw_put(lcwcb);
287                 }
288         }
289
290         cfs_complete(&lcw_stop_completion);
291
292         RETURN(rc);
293 }
294
295 static void lcw_dispatch_start(void)
296 {
297         int rc;
298
299         ENTRY;
300         LASSERT(lcw_refcount == 1);
301
302         cfs_init_completion(&lcw_stop_completion);
303         cfs_init_completion(&lcw_start_completion);
304         cfs_waitq_init(&lcw_event_waitq);
305
306         CDEBUG(D_INFO, "starting dispatch thread\n");
307         rc = cfs_kernel_thread(lcw_dispatch_main, NULL, 0);
308         if (rc < 0) {
309                 CERROR("error spawning watchdog dispatch thread: %d\n", rc);
310                 EXIT;
311                 return;
312         }
313         cfs_wait_for_completion(&lcw_start_completion);
314         CDEBUG(D_INFO, "watchdog dispatcher initialization complete.\n");
315
316         EXIT;
317 }
318
319 static void lcw_dispatch_stop(void)
320 {
321         ENTRY;
322         LASSERT(lcw_refcount == 0);
323
324         CDEBUG(D_INFO, "trying to stop watchdog dispatcher.\n");
325
326         cfs_set_bit(LCW_FLAG_STOP, &lcw_flags);
327         cfs_waitq_signal(&lcw_event_waitq);
328
329         cfs_wait_for_completion(&lcw_stop_completion);
330
331         CDEBUG(D_INFO, "watchdog dispatcher has shut down.\n");
332
333         EXIT;
334 }
335
336 struct lc_watchdog *lc_watchdog_add(int timeout,
337                                     void (*callback)(pid_t, void *),
338                                     void *data)
339 {
340         struct lc_watchdog *lcw = NULL;
341         ENTRY;
342
343         LIBCFS_ALLOC(lcw, sizeof(*lcw));
344         if (lcw == NULL) {
345                 CDEBUG(D_INFO, "Could not allocate new lc_watchdog\n");
346                 RETURN(ERR_PTR(-ENOMEM));
347         }
348
349         lcw->lcw_task     = cfs_current();
350         lcw->lcw_pid      = cfs_curproc_pid();
351         lcw->lcw_callback = (callback != NULL) ? callback : lc_watchdog_dumplog;
352         lcw->lcw_data     = data;
353         lcw->lcw_state    = LC_WATCHDOG_DISABLED;
354
355         CFS_INIT_LIST_HEAD(&lcw->lcw_list);
356         cfs_timer_init(&lcw->lcw_timer, lcw_cb, lcw);
357         cfs_atomic_set(&lcw->lcw_refcount, 1);
358
359         cfs_down(&lcw_refcount_sem);
360         if (++lcw_refcount == 1)
361                 lcw_dispatch_start();
362         cfs_up(&lcw_refcount_sem);
363
364         /* Keep this working in case we enable them by default */
365         if (lcw->lcw_state == LC_WATCHDOG_ENABLED) {
366                 lcw->lcw_last_touched = cfs_time_current();
367                 cfs_timer_arm(&lcw->lcw_timer, cfs_time_seconds(timeout) +
368                               cfs_time_current());
369         }
370
371         RETURN(lcw);
372 }
373 EXPORT_SYMBOL(lc_watchdog_add);
374
375 static void lcw_update_time(struct lc_watchdog *lcw, const char *message)
376 {
377         cfs_time_t newtime = cfs_time_current();;
378
379         if (lcw->lcw_state == LC_WATCHDOG_EXPIRED) {
380                 struct timeval timediff;
381                 cfs_time_t delta_time = cfs_time_sub(newtime,
382                                                      lcw->lcw_last_touched);
383                 cfs_duration_usec(delta_time, &timediff);
384
385                 LCONSOLE_WARN("Service thread pid %u %s after %lu.%.02lus. "
386                               "This indicates the system was overloaded (too "
387                               "many service threads, or there were not enough "
388                               "hardware resources).\n",
389                               lcw->lcw_pid,
390                               message,
391                               timediff.tv_sec,
392                               timediff.tv_usec / 10000);
393         }
394         lcw->lcw_last_touched = newtime;
395 }
396
397 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
398 {
399         ENTRY;
400         LASSERT(lcw != NULL);
401         LASSERT(cfs_atomic_read(&lcw->lcw_refcount) > 0);
402
403         cfs_spin_lock_bh(&lcw_pending_timers_lock);
404         cfs_list_del_init(&lcw->lcw_list);
405         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
406
407         lcw_update_time(lcw, "resumed");
408         lcw->lcw_state = LC_WATCHDOG_ENABLED;
409
410         cfs_timer_arm(&lcw->lcw_timer, cfs_time_current() +
411                       cfs_time_seconds(timeout));
412
413         EXIT;
414 }
415 EXPORT_SYMBOL(lc_watchdog_touch);
416
417 void lc_watchdog_disable(struct lc_watchdog *lcw)
418 {
419         ENTRY;
420         LASSERT(lcw != NULL);
421         LASSERT(cfs_atomic_read(&lcw->lcw_refcount) > 0);
422
423         cfs_spin_lock_bh(&lcw_pending_timers_lock);
424         if (!cfs_list_empty(&lcw->lcw_list))
425                 cfs_list_del_init(&lcw->lcw_list);
426         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
427
428         lcw_update_time(lcw, "completed");
429         lcw->lcw_state = LC_WATCHDOG_DISABLED;
430
431         EXIT;
432 }
433 EXPORT_SYMBOL(lc_watchdog_disable);
434
435 void lc_watchdog_delete(struct lc_watchdog *lcw)
436 {
437         ENTRY;
438         LASSERT(lcw != NULL);
439         LASSERT(cfs_atomic_read(&lcw->lcw_refcount) > 0);
440
441         cfs_timer_disarm(&lcw->lcw_timer);
442
443         lcw_update_time(lcw, "stopped");
444
445         cfs_spin_lock_bh(&lcw_pending_timers_lock);
446         if (!cfs_list_empty(&lcw->lcw_list))
447                 cfs_list_del_init(&lcw->lcw_list);
448         cfs_spin_unlock_bh(&lcw_pending_timers_lock);
449         lcw_put(lcw);
450
451         cfs_down(&lcw_refcount_sem);
452         if (--lcw_refcount == 0)
453                 lcw_dispatch_stop();
454         cfs_up(&lcw_refcount_sem);
455
456         EXIT;
457 }
458 EXPORT_SYMBOL(lc_watchdog_delete);
459
460 /*
461  * Provided watchdog handlers
462  */
463
464 void lc_watchdog_dumplog(pid_t pid, void *data)
465 {
466         libcfs_debug_dumplog_internal((void *)((long_ptr_t)pid));
467 }
468 EXPORT_SYMBOL(lc_watchdog_dumplog);
469
470 #else   /* !defined(WITH_WATCHDOG) */
471
472 struct lc_watchdog *lc_watchdog_add(int timeout,
473                                     void (*callback)(pid_t pid, void *),
474                                     void *data)
475 {
476         static struct lc_watchdog      watchdog;
477         return &watchdog;
478 }
479 EXPORT_SYMBOL(lc_watchdog_add);
480
481 void lc_watchdog_touch(struct lc_watchdog *lcw, int timeout)
482 {
483 }
484 EXPORT_SYMBOL(lc_watchdog_touch);
485
486 void lc_watchdog_disable(struct lc_watchdog *lcw)
487 {
488 }
489 EXPORT_SYMBOL(lc_watchdog_disable);
490
491 void lc_watchdog_delete(struct lc_watchdog *lcw)
492 {
493 }
494 EXPORT_SYMBOL(lc_watchdog_delete);
495
496 #endif