libcfs/libcfs/debug.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  *
  31  * libcfs/libcfs/debug.c
  32  *
  33  * Author: Phil Schwan <phil@clusterfs.com>
  34  *
  35  */
  36
  37 # define DEBUG_SUBSYSTEM S_LNET
  38
  39 #include <linux/module.h>
  40 #include <linux/ctype.h>
  41 #include <libcfs/libcfs_string.h>
  42 #include <linux/kthread.h>
  43 #include <linux/stacktrace.h>
  44 #include <linux/utsname.h>
  45 #include <linux/kallsyms.h>
  46 #ifdef HAVE_PANIC_NOTIFIER_H
  47 #include <linux/panic_notifier.h>
  48 #endif
  49 #include "tracefile.h"
  50
  51 static char debug_file_name[1024];
  52
  53 unsigned int libcfs_subsystem_debug = ~0;
  54 EXPORT_SYMBOL(libcfs_subsystem_debug);
  55 module_param(libcfs_subsystem_debug, int, 0644);
  56 MODULE_PARM_DESC(libcfs_subsystem_debug, "Lustre kernel debug subsystem mask");
  57
  58 unsigned int libcfs_debug = (D_CANTMASK | D_NETERROR | D_HA | D_CONFIG |
  59                              D_IOCTL | D_LFSCK);
  60 EXPORT_SYMBOL(libcfs_debug);
  61 module_param(libcfs_debug, int, 0644);
  62 MODULE_PARM_DESC(libcfs_debug, "Lustre kernel debug mask");
  63
  64 static int libcfs_param_debug_mb_set(const char *val,
  65                                      cfs_kernel_param_arg_t *kp)
  66 {
  67         int rc;
  68         unsigned int num;
  69
  70         rc = kstrtouint(val, 0, &num);
  71         if (rc < 0)
  72                 return rc;
  73
  74         num = cfs_trace_set_debug_mb(num);
  75
  76         *((unsigned int *)kp->arg) = num;
  77         num = cfs_trace_get_debug_mb();
  78         if (num)
  79                 /* This value is more precise */
  80                 *((unsigned int *)kp->arg) = num;
  81
  82         return 0;
  83 }
  84
  85 /* While debug_mb setting look like unsigned int, in fact
  86  * it needs quite a bunch of extra processing, so we define special
  87  * debug_mb parameter type with corresponding methods to handle this case
  88  */
  89 static const struct kernel_param_ops param_ops_debug_mb = {
  90         .set = libcfs_param_debug_mb_set,
  91         .get = param_get_uint,
  92 };
  93
  94 #define param_check_debug_mb(name, p) \
  95                 __param_check(name, p, unsigned int)
  96
  97 static unsigned int libcfs_debug_mb;
  98 #ifdef HAVE_KERNEL_PARAM_OPS
  99 module_param(libcfs_debug_mb, debug_mb, 0644);
 100 #else
 101 module_param_call(libcfs_debug_mb, libcfs_param_debug_mb_set, param_get_uint,
 102                   &param_ops_debug_mb, 0644);
 103 #endif
 104 MODULE_PARM_DESC(libcfs_debug_mb, "Total debug buffer size.");
 105
 106 unsigned int libcfs_printk = D_CANTMASK;
 107 module_param(libcfs_printk, uint, 0644);
 108 MODULE_PARM_DESC(libcfs_printk, "Lustre kernel debug console mask");
 109
 110 unsigned int libcfs_console_ratelimit = 1;
 111 module_param(libcfs_console_ratelimit, uint, 0644);
 112 MODULE_PARM_DESC(libcfs_console_ratelimit, "Lustre kernel debug console ratelimit (0 to disable)");
 113
 114 static int param_set_delay_minmax(const char *val,
 115                                   cfs_kernel_param_arg_t *kp,
 116                                   long min, long max)
 117 {
 118         long d;
 119         int sec;
 120         int rc;
 121
 122         rc = kstrtoint(val, 0, &sec);
 123         if (rc)
 124                 return -EINVAL;
 125
 126         /* The sysfs setting is in centiseconds */
 127         d = cfs_time_seconds(sec) / 100;
 128         if (d < min || d > max)
 129                 return -EINVAL;
 130
 131         *((unsigned int *)kp->arg) = d;
 132
 133         return 0;
 134 }
 135
 136 static int param_get_delay(char *buffer, cfs_kernel_param_arg_t *kp)
 137 {
 138         unsigned int d = *(unsigned int *)kp->arg;
 139
 140         param_get_byte(buffer, kp);
 141         return sprintf(buffer, "%lu%c", jiffies_to_msecs(d * 10) / MSEC_PER_SEC,
 142                        strnchr(buffer, PAGE_SIZE, '\n') ? '\n' : '\0');
 143 }
 144
 145 unsigned int libcfs_console_max_delay;
 146 unsigned int libcfs_console_min_delay;
 147
 148 static int param_set_console_max_delay(const char *val,
 149                                        cfs_kernel_param_arg_t *kp)
 150 {
 151         return param_set_delay_minmax(val, kp,
 152                                       libcfs_console_min_delay, INT_MAX);
 153 }
 154
 155 static const struct kernel_param_ops param_ops_console_max_delay = {
 156         .set = param_set_console_max_delay,
 157         .get = param_get_delay,
 158 };
 159
 160 #define param_check_console_max_delay(name, p) \
 161                 __param_check(name, p, unsigned int)
 162
 163 #ifdef HAVE_KERNEL_PARAM_OPS
 164 module_param(libcfs_console_max_delay, console_max_delay, 0644);
 165 #else
 166 module_param_call(libcfs_console_max_delay, param_set_console_max_delay,
 167                   param_get_delay, &param_ops_console_max_delay, 0644);
 168 #endif
 169 MODULE_PARM_DESC(libcfs_console_max_delay, "Lustre kernel debug console max delay (jiffies)");
 170
 171 static int param_set_console_min_delay(const char *val,
 172                                        cfs_kernel_param_arg_t *kp)
 173 {
 174         return param_set_delay_minmax(val, kp,
 175                                       1, libcfs_console_max_delay);
 176 }
 177
 178 static const struct kernel_param_ops param_ops_console_min_delay = {
 179         .set = param_set_console_min_delay,
 180         .get = param_get_delay,
 181 };
 182
 183 #define param_check_console_min_delay(name, p) \
 184                 __param_check(name, p, unsigned int)
 185
 186 #ifdef HAVE_KERNEL_PARAM_OPS
 187 module_param(libcfs_console_min_delay, console_min_delay, 0644);
 188 #else
 189 module_param_call(libcfs_console_min_delay, param_set_console_min_delay,
 190                   param_get_delay, &param_ops_console_min_delay, 0644);
 191 #endif
 192 MODULE_PARM_DESC(libcfs_console_min_delay, "Lustre kernel debug console min delay (jiffies)");
 193
 194 #ifndef HAVE_PARAM_SET_UINT_MINMAX
 195 static int param_set_uint_minmax(const char *val,
 196                                  cfs_kernel_param_arg_t *kp,
 197                                  unsigned int min, unsigned int max)
 198 {
 199         unsigned int num;
 200         int ret;
 201
 202         if (!val)
 203                 return -EINVAL;
 204
 205         ret = kstrtouint(val, 0, &num);
 206         if (ret < 0 || num < min || num > max)
 207                 return -EINVAL;
 208
 209         *((unsigned int *)kp->arg) = num;
 210         return 0;
 211 }
 212 #endif
 213
 214 static int param_set_uintpos(const char *val,
 215                              cfs_kernel_param_arg_t *kp)
 216 {
 217         return param_set_uint_minmax(val, kp, 1, -1);
 218 }
 219
 220 static const struct kernel_param_ops param_ops_uintpos = {
 221         .set = param_set_uintpos,
 222         .get = param_get_uint,
 223 };
 224
 225 #define param_check_uintpos(name, p) \
 226                 __param_check(name, p, unsigned int)
 227
 228 unsigned int libcfs_console_backoff = CDEBUG_DEFAULT_BACKOFF;
 229 #ifdef HAVE_KERNEL_PARAM_OPS
 230 module_param(libcfs_console_backoff, uintpos, 0644);
 231 #else
 232 module_param_call(libcfs_console_backoff, param_set_uintpos, param_get_uint,
 233                   &param_ops_uintpos, 0644);
 234 #endif
 235 MODULE_PARM_DESC(libcfs_console_backoff, "Lustre kernel debug console backoff factor");
 236
 237 unsigned int libcfs_debug_binary = 1;
 238
 239 unsigned int libcfs_stack = 3 * THREAD_SIZE / 4;
 240 EXPORT_SYMBOL(libcfs_stack);
 241
 242 unsigned int libcfs_catastrophe;
 243 EXPORT_SYMBOL(libcfs_catastrophe);
 244
 245 unsigned int libcfs_watchdog_ratelimit = 300;
 246 EXPORT_SYMBOL(libcfs_watchdog_ratelimit);
 247
 248 unsigned int libcfs_panic_on_lbug = 1;
 249 module_param(libcfs_panic_on_lbug, uint, 0644);
 250 MODULE_PARM_DESC(libcfs_panic_on_lbug, "Lustre kernel panic on LBUG");
 251
 252 atomic64_t libcfs_kmem = ATOMIC64_INIT(0);
 253 EXPORT_SYMBOL(libcfs_kmem);
 254
 255 static DECLARE_COMPLETION(debug_complete);
 256
 257 /* We need to pass a pointer here, but elsewhere this must be a const */
 258 char *libcfs_debug_file_path = LIBCFS_DEBUG_FILE_PATH_DEFAULT;
 259 EXPORT_SYMBOL(libcfs_debug_file_path);
 260 module_param(libcfs_debug_file_path, charp, 0644);
 261 MODULE_PARM_DESC(libcfs_debug_file_path,
 262                  "Path for dumping debug logs, set 'NONE' to prevent log dumping");
 263
 264 int libcfs_panic_in_progress;
 265
 266 /* libcfs_debug_token2mask() expects the returned string in lower-case */
 267 static const char *libcfs_debug_subsys2str(int subsys)
 268 {
 269         static const char * const libcfs_debug_subsystems[] =
 270                 LIBCFS_DEBUG_SUBSYS_NAMES;
 271
 272         if (subsys >= ARRAY_SIZE(libcfs_debug_subsystems))
 273                 return NULL;
 274
 275         return libcfs_debug_subsystems[subsys];
 276 }
 277
 278 /* libcfs_debug_token2mask() expects the returned string in lower-case */
 279 static const char *libcfs_debug_dbg2str(int debug)
 280 {
 281         static const char * const libcfs_debug_masks[] =
 282                 LIBCFS_DEBUG_MASKS_NAMES;
 283
 284         if (debug >= ARRAY_SIZE(libcfs_debug_masks))
 285                 return NULL;
 286
 287         return libcfs_debug_masks[debug];
 288 }
 289
 290 int
 291 libcfs_debug_mask2str(char *str, int size, int mask, int is_subsys)
 292 {
 293         const char *(*fn)(int bit) = is_subsys ? libcfs_debug_subsys2str :
 294                                                  libcfs_debug_dbg2str;
 295         int len = 0;
 296         const char *token;
 297         int i;
 298
 299         if (mask == 0) {                        /* "0" */
 300                 if (size > 0)
 301                         str[0] = '0';
 302                 len = 1;
 303         } else {                                /* space-separated tokens */
 304                 for (i = 0; i < 32; i++) {
 305                         if ((mask & BIT(i)) == 0)
 306                                 continue;
 307
 308                         token = fn(i);
 309                         if (!token)     /* unused bit */
 310                                 continue;
 311
 312                         if (len > 0) {          /* separator? */
 313                                 if (len < size)
 314                                         str[len] = ' ';
 315                                 len++;
 316                         }
 317
 318                         while (*token != 0) {
 319                                 if (len < size)
 320                                         str[len] = *token;
 321                                 token++;
 322                                 len++;
 323                         }
 324                 }
 325         }
 326
 327         /* terminate 'str' */
 328         if (len < size)
 329                 str[len] = 0;
 330         else
 331                 str[size - 1] = 0;
 332
 333         return len;
 334 }
 335
 336 int
 337 libcfs_debug_str2mask(int *mask, const char *str, int is_subsys)
 338 {
 339         const char *(*fn)(int bit) = is_subsys ? libcfs_debug_subsys2str :
 340                                                  libcfs_debug_dbg2str;
 341         int m = 0;
 342         int matched;
 343         int n;
 344         int t;
 345
 346         /* Allow a number for backwards compatibility */
 347         for (n = strlen(str); n > 0; n--)
 348                 if (!isspace(str[n-1]))
 349                         break;
 350         matched = n;
 351         t = sscanf(str, "%i%n", &m, &matched);
 352         if (t >= 1 && matched == n) {
 353                 /* don't print warning for lctl set_param debug=0 or -1 */
 354                 if (m != 0 && m != -1)
 355                         CWARN("You are trying to use a numerical value for the mask - this will be deprecated in a future release.\n");
 356                 *mask = m;
 357                 return 0;
 358         }
 359
 360         return cfs_str2mask(str, fn, mask, is_subsys ? 0 : D_CANTMASK,
 361                             0xffffffff);
 362 }
 363
 364 char lnet_debug_log_upcall[1024] = "/usr/lib/lustre/lnet_debug_log_upcall";
 365
 366 /* Upcall function once a Lustre log has been dumped.
 367  *
 368  * @file        path of the dumped log
 369  */
 370 static void libcfs_run_debug_log_upcall(char *file)
 371 {
 372         char *argv[3];
 373         int rc;
 374         static const char * const envp[] = {
 375                 "HOME=/",
 376                 "PATH=/sbin:/bin:/usr/sbin:/usr/bin",
 377                 NULL
 378         };
 379
 380         ENTRY;
 381         argv[0] = lnet_debug_log_upcall;
 382
 383         LASSERTF(file, "called on a null filename\n");
 384         argv[1] = file; /* only need to pass the path of the file */
 385
 386         argv[2] = NULL;
 387
 388         rc = call_usermodehelper(argv[0], argv, (char **)envp, 1);
 389         if (rc < 0 && rc != -ENOENT) {
 390                 CERROR("Error %d invoking LNET debug log upcall %s %s; check /sys/kernel/debug/lnet/debug_log_upcall\n",
 391                        rc, argv[0], argv[1]);
 392         } else {
 393                 CDEBUG(D_HA, "Invoked LNET debug log upcall %s %s\n",
 394                        argv[0], argv[1]);
 395         }
 396 }
 397
 398 /**
 399  * Dump Lustre log to ::debug_file_path by calling tracefile_dump_all_pages()
 400  */
 401 static void libcfs_debug_dumplog_internal(void *arg)
 402 {
 403         static time64_t last_dump_time;
 404         time64_t current_time;
 405
 406         current_time = ktime_get_real_seconds();
 407
 408         if (strncmp(libcfs_debug_file_path, "NONE", 4) != 0 &&
 409             current_time > last_dump_time) {
 410                 last_dump_time = current_time;
 411                 snprintf(debug_file_name, sizeof(debug_file_name) - 1,
 412                          "%s.%lld.%ld", libcfs_debug_file_path,
 413                          (s64)current_time, (uintptr_t)arg);
 414                 pr_alert("LustreError: dumping log to %s\n", debug_file_name);
 415                 cfs_tracefile_dump_all_pages(debug_file_name);
 416                 libcfs_run_debug_log_upcall(debug_file_name);
 417         }
 418 }
 419
 420 static int libcfs_debug_dumplog_thread(void *arg)
 421 {
 422         libcfs_debug_dumplog_internal(arg);
 423         complete(&debug_complete);
 424         return 0;
 425 }
 426
 427 static DEFINE_MUTEX(libcfs_debug_dumplog_lock);
 428
 429 void libcfs_debug_dumplog(void)
 430 {
 431         struct task_struct *dumper;
 432
 433         ENTRY;
 434
 435         if (mutex_trylock(&libcfs_debug_dumplog_lock) == 0)
 436                 return;
 437
 438         /* If a previous call was interrupted, debug_complete->done
 439          * might be elevated, and so we won't actually wait here.
 440          * So we reinit the completion to ensure we wait for
 441          * one thread to complete, though it might not be the one
 442          * we start if there are overlaping thread.
 443          */
 444         reinit_completion(&debug_complete);
 445         dumper = kthread_run(libcfs_debug_dumplog_thread,
 446                              (void *)(long)current->pid,
 447                              "libcfs_debug_dumper");
 448         if (IS_ERR(dumper))
 449                 pr_err("LustreError: cannot start log dump thread: rc = %ld\n",
 450                        PTR_ERR(dumper));
 451         else
 452                 wait_for_completion_interruptible(&debug_complete);
 453
 454         mutex_unlock(&libcfs_debug_dumplog_lock);
 455 }
 456 EXPORT_SYMBOL(libcfs_debug_dumplog);
 457
 458 /* coverity[+kill] */
 459 void __noreturn lbug_with_loc(struct libcfs_debug_msg_data *msgdata)
 460 {
 461         libcfs_catastrophe = 1;
 462         libcfs_debug_msg(msgdata, "LBUG\n");
 463
 464         if (in_interrupt()) {
 465                 panic("LBUG in interrupt.\n");
 466                 /* not reached */
 467         }
 468
 469         libcfs_debug_dumpstack(NULL);
 470         if (libcfs_panic_on_lbug)
 471                 panic("LBUG");
 472         else
 473                 libcfs_debug_dumplog();
 474         set_current_state(TASK_UNINTERRUPTIBLE);
 475         while (1)
 476                 schedule();
 477 }
 478 EXPORT_SYMBOL(lbug_with_loc);
 479
 480 #ifdef CONFIG_STACKTRACE
 481
 482 #ifndef HAVE_SAVE_STACK_TRACE_TSK
 483 #define save_stack_trace_tsk(tsk, trace)                                       \
 484 do {                                                                           \
 485         if (tsk == current)                                                    \
 486                 save_stack_trace(trace);                                       \
 487         else                                                                   \
 488                 pr_info("No stack, save_stack_trace_tsk() not exported\n");    \
 489 } while (0)
 490 #endif
 491
 492 static void cfs_print_stack_trace(unsigned long *entries, unsigned int nr)
 493 {
 494         unsigned int i;
 495
 496         /* Prefer %pB for backtraced symbolic names since it was added in:
 497          * Linux v2.6.38-6557-g0f77a8d37825
 498          * vsprintf: Introduce %pB format specifier
 499          */
 500         for (i = 0; i < nr; i++)
 501                 pr_info("[<0>] %pB\n", (void *)entries[i]);
 502 }
 503
 504 #define MAX_ST_ENTRIES  100
 505 static DEFINE_SPINLOCK(st_lock);
 506
 507 /* Linux v5.1-rc5 214d8ca6ee ("stacktrace: Provide common infrastructure")
 508  * CONFIG_ARCH_STACKWALK indicates that save_stack_trace_tsk symbol is not
 509  * exported. Use symbol_get() to find if save_stack_trace_tsk is available.
 510  */
 511 #ifdef CONFIG_ARCH_STACKWALK
 512 typedef unsigned int (stack_trace_save_tsk_t)(struct task_struct *task,
 513                                               unsigned long *store,
 514                                               unsigned int size,
 515                                               unsigned int skipnr);
 516 static stack_trace_save_tsk_t *task_dump_stack;
 517 #endif
 518
 519 void __init cfs_debug_init(void)
 520 {
 521 #ifdef CONFIG_ARCH_STACKWALK
 522         task_dump_stack = (void *)
 523                         cfs_kallsyms_lookup_name("stack_trace_save_tsk");
 524
 525 #endif
 526 }
 527
 528 static void libcfs_call_trace(struct task_struct *tsk)
 529 {
 530         static unsigned long entries[MAX_ST_ENTRIES];
 531 #ifdef CONFIG_ARCH_STACKWALK
 532         unsigned int nr_entries;
 533
 534         spin_lock(&st_lock);
 535         pr_info("Pid: %d, comm: %.20s %s %s\n", tsk->pid, tsk->comm,
 536                 init_utsname()->release, init_utsname()->version);
 537         pr_info("Call Trace TBD:\n");
 538         if (task_dump_stack) {
 539                 nr_entries = task_dump_stack(tsk, entries, MAX_ST_ENTRIES, 0);
 540                 cfs_print_stack_trace(entries, nr_entries);
 541         }
 542         spin_unlock(&st_lock);
 543 #else
 544         struct stack_trace trace;
 545
 546         trace.nr_entries = 0;
 547         trace.max_entries = MAX_ST_ENTRIES;
 548         trace.entries = entries;
 549         trace.skip = 0;
 550
 551         spin_lock(&st_lock);
 552         pr_info("Pid: %d, comm: %.20s %s %s\n", tsk->pid, tsk->comm,
 553                 init_utsname()->release, init_utsname()->version);
 554         pr_info("Call Trace:\n");
 555         save_stack_trace_tsk(tsk, &trace);
 556         cfs_print_stack_trace(trace.entries, trace.nr_entries);
 557         spin_unlock(&st_lock);
 558 #endif
 559 }
 560
 561 #else /* !CONFIG_STACKTRACE */
 562
 563 #ifdef CONFIG_X86
 564 #include <linux/nmi.h>
 565 #include <asm/stacktrace.h>
 566
 567 #ifdef HAVE_STACKTRACE_OPS
 568 static int print_trace_stack(void *data, char *name)
 569 {
 570         printk(" <%s> ", name);
 571         return 0;
 572 }
 573
 574 #ifdef STACKTRACE_OPS_ADDRESS_RETURN_INT
 575 static int
 576 #else
 577 static void
 578 #endif
 579 print_trace_address(void *data, unsigned long addr, int reliable)
 580 {
 581         char fmt[32];
 582
 583         touch_nmi_watchdog();
 584         sprintf(fmt, " [<%016lx>] %s%%s\n", addr, reliable ? "" : "? ");
 585         __print_symbol(fmt, addr);
 586 #ifdef STACKTRACE_OPS_ADDRESS_RETURN_INT
 587         return 0;
 588 #endif
 589 }
 590
 591 static const struct stacktrace_ops print_trace_ops = {
 592         .stack          = print_trace_stack,
 593         .address        = print_trace_address,
 594         .walk_stack     = print_context_stack,
 595 };
 596 #endif /* HAVE_STACKTRACE_OPS */
 597
 598 static void libcfs_call_trace(struct task_struct *tsk)
 599 {
 600 #ifdef HAVE_STACKTRACE_OPS
 601         printk("Pid: %d, comm: %.20s\n", tsk->pid, tsk->comm);
 602         printk("\nCall Trace:\n");
 603         dump_trace(tsk, NULL, NULL, 0, &print_trace_ops, NULL);
 604         printk("\n");
 605 #else /* !HAVE_STACKTRACE_OPS */
 606         if (tsk == current)
 607                 dump_stack();
 608         else
 609                 CWARN("can't show stack: kernel doesn't export show_task\n");
 610 #endif /* HAVE_STACKTRACE_OPS */
 611 }
 612
 613 #else /* !CONFIG_X86 */
 614
 615 static void libcfs_call_trace(struct task_struct *tsk)
 616 {
 617         if (tsk == current)
 618                 dump_stack();
 619         else
 620                 CWARN("can't show stack: kernel doesn't export show_task\n");
 621 }
 622
 623 #endif /* CONFIG_X86 */
 624
 625 #endif /* CONFIG_STACKTRACE */
 626
 627 void libcfs_debug_dumpstack(struct task_struct *tsk)
 628 {
 629         libcfs_call_trace(tsk ?: current);
 630 }
 631 EXPORT_SYMBOL(libcfs_debug_dumpstack);
 632
 633 static int panic_notifier(struct notifier_block *self, unsigned long unused1,
 634                           void *unused2)
 635 {
 636         if (libcfs_panic_in_progress)
 637                 return 0;
 638
 639         libcfs_panic_in_progress = 1;
 640         mb();
 641
 642 #ifdef LNET_DUMP_ON_PANIC
 643         /* This is currently disabled because it spews far too much to the
 644          * console on the rare cases it is ever triggered. */
 645
 646         if (in_interrupt()) {
 647                 cfs_trace_debug_print();
 648         } else {
 649                 libcfs_debug_dumplog_internal((void *)(long)current->pid);
 650         }
 651 #endif
 652         return 0;
 653 }
 654
 655 static struct notifier_block libcfs_panic_notifier = {
 656         .notifier_call          = panic_notifier,
 657         .next                   = NULL,
 658         .priority               = 10000,
 659 };
 660
 661 static void libcfs_register_panic_notifier(void)
 662 {
 663         atomic_notifier_chain_register(&panic_notifier_list,
 664                                        &libcfs_panic_notifier);
 665 }
 666
 667 static void libcfs_unregister_panic_notifier(void)
 668 {
 669         atomic_notifier_chain_unregister(&panic_notifier_list,
 670                                          &libcfs_panic_notifier);
 671 }
 672
 673 int libcfs_debug_init(unsigned long bufsize)
 674 {
 675         unsigned int max = libcfs_debug_mb;
 676         int rc = 0;
 677
 678         if (libcfs_console_max_delay <= 0 || /* not set by user or */
 679             libcfs_console_min_delay <= 0 || /* set to invalid values */
 680             libcfs_console_min_delay >= libcfs_console_max_delay) {
 681                 libcfs_console_max_delay = CDEBUG_DEFAULT_MAX_DELAY;
 682                 libcfs_console_min_delay = CDEBUG_DEFAULT_MIN_DELAY;
 683         }
 684
 685         /* If libcfs_debug_mb is uninitialized then just make the
 686          * total buffers smp_num_cpus * TCD_MAX_PAGES
 687          */
 688         if (max < num_possible_cpus()) {
 689                 max = TCD_MAX_PAGES;
 690         } else {
 691                 max = (max / num_possible_cpus());
 692                 max <<= (20 - PAGE_SHIFT);
 693         }
 694
 695         rc = cfs_tracefile_init(max);
 696         if (rc)
 697                 return rc;
 698
 699         libcfs_register_panic_notifier();
 700         kernel_param_lock(THIS_MODULE);
 701         if (libcfs_debug_mb == 0)
 702                 libcfs_debug_mb = cfs_trace_get_debug_mb();
 703         kernel_param_unlock(THIS_MODULE);
 704         return rc;
 705 }
 706
 707 int libcfs_debug_cleanup(void)
 708 {
 709         libcfs_unregister_panic_notifier();
 710         kernel_param_lock(THIS_MODULE);
 711         cfs_tracefile_exit();
 712         kernel_param_unlock(THIS_MODULE);
 713         return 0;
 714 }
 715
 716 int libcfs_debug_clear_buffer(void)
 717 {
 718         cfs_trace_flush_pages();
 719         return 0;
 720 }
 721
 722 /* Debug markers, although printed by S_LNET should not be be marked as such. */
 723 #undef DEBUG_SUBSYSTEM
 724 #define DEBUG_SUBSYSTEM S_UNDEFINED
 725 int libcfs_debug_mark_buffer(const char *text)
 726 {
 727         CDEBUG(D_TRACE,
 728                "**************************************************\n");
 729         LCONSOLE(D_WARNING, "DEBUG MARKER: %s\n", text);
 730         CDEBUG(D_TRACE,
 731                "**************************************************\n");
 732
 733         return 0;
 734 }
 735
 736 #undef DEBUG_SUBSYSTEM
 737 #define DEBUG_SUBSYSTEM S_LNET