lnet/libcfs/debug.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (C) 2002 Cluster File Systems, Inc.
   5  *   Author: Phil Schwan <phil@clusterfs.com>
   6  *
   7  *   This file is part of Lustre, http://www.lustre.org.
   8  *
   9  *   Lustre is free software; you can redistribute it and/or
  10  *   modify it under the terms of version 2 of the GNU General Public
  11  *   License as published by the Free Software Foundation.
  12  *
  13  *   Lustre is distributed in the hope that it will be useful,
  14  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  *   GNU General Public License for more details.
  17  *
  18  *   You should have received a copy of the GNU General Public License
  19  *   along with Lustre; if not, write to the Free Software
  20  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21  */
  22
  23 #ifndef EXPORT_SYMTAB
  24 # define EXPORT_SYMTAB
  25 #endif
  26
  27 # define DEBUG_SUBSYSTEM S_LNET
  28
  29 #include <libcfs/kp30.h>
  30 #include <libcfs/libcfs.h>
  31 #include "tracefile.h"
  32
  33 static char debug_file_name[1024];
  34
  35 #ifdef __KERNEL__
  36 unsigned int libcfs_subsystem_debug = ~0;
  37 EXPORT_SYMBOL(libcfs_subsystem_debug);
  38
  39 unsigned int libcfs_debug = (D_EMERG | D_ERROR | D_WARNING | D_CONSOLE |
  40                              D_NETERROR | D_HA | D_CONFIG | D_IOCTL |
  41                              D_DLMTRACE | D_RPCTRACE | D_VFSTRACE);
  42 EXPORT_SYMBOL(libcfs_debug);
  43
  44 unsigned int libcfs_printk;
  45 EXPORT_SYMBOL(libcfs_printk);
  46
  47 unsigned int libcfs_console_ratelimit = 1;
  48 EXPORT_SYMBOL(libcfs_console_ratelimit);
  49
  50 unsigned int libcfs_debug_binary = 1;
  51 EXPORT_SYMBOL(libcfs_debug_binary);
  52
  53 unsigned int libcfs_stack;
  54 EXPORT_SYMBOL(libcfs_stack);
  55
  56 unsigned int portal_enter_debugger;
  57 EXPORT_SYMBOL(portal_enter_debugger);
  58
  59 unsigned int libcfs_catastrophe;
  60 EXPORT_SYMBOL(libcfs_catastrophe);
  61
  62 atomic_t libcfs_kmemory = ATOMIC_INIT(0);
  63 EXPORT_SYMBOL(libcfs_kmemory);
  64
  65 static cfs_waitq_t debug_ctlwq;
  66
  67 char debug_file_path[1024] = "/tmp/lustre-log";
  68
  69 int libcfs_panic_in_progress;
  70
  71 /* libcfs_debug_token2mask() expects the returned
  72  * string in lower-case */
  73 const char *
  74 libcfs_debug_subsys2str(int subsys)
  75 {
  76         switch (subsys) {
  77         default:
  78                 return NULL;
  79         case S_UNDEFINED:
  80                 return "undefined";
  81         case S_MDC:
  82                 return "mdc";
  83         case S_MDS:
  84                 return "mds";
  85         case S_OSC:
  86                 return "osc";
  87         case S_OST:
  88                 return "ost";
  89         case S_CLASS:
  90                 return "class";
  91         case S_LOG:
  92                 return "log";
  93         case S_LLITE:
  94                 return "llite";
  95         case S_RPC:
  96                 return "rpc";
  97         case S_LNET:
  98                 return "lnet";
  99         case S_LND:
 100                 return "lnd";
 101         case S_PINGER:
 102                 return "pinger";
 103         case S_FILTER:
 104                 return "filter";
 105         case S_ECHO:
 106                 return "echo";
 107         case S_LDLM:
 108                 return "ldlm";
 109         case S_LOV:
 110                 return "lov";
 111         case S_LMV:
 112                 return "lmv";
 113         case S_SEC:
 114                 return "sec";
 115         case S_GSS:
 116                 return "gss";
 117         case S_MGC:
 118                 return "mgc";
 119         case S_MGS:
 120                 return "mgs";
 121         case S_FID:
 122                 return "fid";
 123         case S_FLD:
 124                 return "fld";
 125         }
 126 }
 127
 128 /* libcfs_debug_token2mask() expects the returned
 129  * string in lower-case */
 130 const char *
 131 libcfs_debug_dbg2str(int debug)
 132 {
 133         switch (debug) {
 134         default:
 135                 return NULL;
 136         case D_TRACE:
 137                 return "trace";
 138         case D_INODE:
 139                 return "inode";
 140         case D_SUPER:
 141                 return "super";
 142         case D_EXT2:
 143                 return "ext2";
 144         case D_MALLOC:
 145                 return "malloc";
 146         case D_CACHE:
 147                 return "cache";
 148         case D_INFO:
 149                 return "info";
 150         case D_IOCTL:
 151                 return "ioctl";
 152         case D_NETERROR:
 153                 return "neterror";
 154         case D_NET:
 155                 return "net";
 156         case D_WARNING:
 157                 return "warning";
 158         case D_BUFFS:
 159                 return "buffs";
 160         case D_OTHER:
 161                 return "other";
 162         case D_DENTRY:
 163                 return "dentry";
 164         case D_PAGE:
 165                 return "page";
 166         case D_DLMTRACE:
 167                 return "dlmtrace";
 168         case D_ERROR:
 169                 return "error";
 170         case D_EMERG:
 171                 return "emerg";
 172         case D_HA:
 173                 return "ha";
 174         case D_RPCTRACE:
 175                 return "rpctrace";
 176         case D_VFSTRACE:
 177                 return "vfstrace";
 178         case D_READA:
 179                 return "reada";
 180         case D_MMAP:
 181                 return "mmap";
 182         case D_CONFIG:
 183                 return "config";
 184         case D_CONSOLE:
 185                 return "console";
 186         case D_QUOTA:
 187                 return "quota";
 188         case D_SEC:
 189                 return "sec";
 190         }
 191 }
 192
 193 int
 194 libcfs_debug_mask2str(char *str, int size, int mask, int is_subsys)
 195 {
 196         const char *(*fn)(int bit) = is_subsys ? libcfs_debug_subsys2str :
 197                                                  libcfs_debug_dbg2str;
 198         int           len = 0;
 199         const char   *token;
 200         int           bit;
 201         int           i;
 202
 203         if (mask == 0) {                        /* "0" */
 204                 if (size > 0)
 205                         str[0] = '0';
 206                 len = 1;
 207         } else {                                /* space-separated tokens */
 208                 for (i = 0; i < 32; i++) {
 209                         bit = 1 << i;
 210
 211                         if ((mask & bit) == 0)
 212                                 continue;
 213
 214                         token = fn(bit);
 215                         if (token == NULL)              /* unused bit */
 216                                 continue;
 217
 218                         if (len > 0) {                  /* separator? */
 219                                 if (len < size)
 220                                         str[len] = ' ';
 221                                 len++;
 222                         }
 223
 224                         while (*token != 0) {
 225                                 if (len < size)
 226                                         str[len] = *token;
 227                                 token++;
 228                                 len++;
 229                         }
 230                 }
 231         }
 232
 233         /* terminate 'str' */
 234         if (len < size)
 235                 str[len] = 0;
 236         else
 237                 str[size - 1] = 0;
 238
 239         return len;
 240 }
 241
 242 int
 243 libcfs_debug_token2mask(int *mask, const char *str, int len, int is_subsys)
 244 {
 245         const char *(*fn)(int bit) = is_subsys ? libcfs_debug_subsys2str :
 246                                                  libcfs_debug_dbg2str;
 247         int           i;
 248         int           j;
 249         int           bit;
 250         const char   *token;
 251
 252         /* match against known tokens */
 253         for (i = 0; i < 32; i++) {
 254                 bit = 1 << i;
 255
 256                 token = fn(bit);
 257                 if (token == NULL)              /* unused? */
 258                         continue;
 259
 260                 /* strcasecmp */
 261                 for (j = 0; ; j++) {
 262                         if (j == len) {         /* end of token */
 263                                 if (token[j] == 0) {
 264                                         *mask = bit;
 265                                         return 0;
 266                                 }
 267                                 break;
 268                         }
 269
 270                         if (token[j] == 0)
 271                                 break;
 272
 273                         if (str[j] == token[j])
 274                                 continue;
 275
 276                         if (str[j] < 'A' || 'Z' < str[j])
 277                                 break;
 278
 279                         if (str[j] - 'A' + 'a' != token[j])
 280                                 break;
 281                 }
 282         }
 283
 284         return -EINVAL;                         /* no match */
 285 }
 286
 287 int
 288 libcfs_debug_str2mask(int *mask, const char *str, int is_subsys)
 289 {
 290         int         m = 0;
 291         int         matched = 0;
 292         char        op = 0;
 293         int         n;
 294         int         t;
 295
 296         /* <str> must be a list of debug tokens or numbers separated by
 297          * whitespace and optionally an operator ('+' or '-').  If an operator
 298          * appears first in <str>, '*mask' is used as the starting point
 299          * (relative), otherwise 0 is used (absolute).  An operator applies to
 300          * all following tokens up to the next operator. */
 301
 302         while (*str != 0) {
 303                 while (isspace(*str)) /* skip whitespace */
 304                         str++;
 305
 306                 if (*str == 0)
 307                         break;
 308
 309                 if (*str == '+' || *str == '-') {
 310                         op = *str++;
 311
 312                         /* op on first token == relative */
 313                         if (!matched)
 314                                 m = *mask;
 315
 316                         while (isspace(*str)) /* skip whitespace */
 317                                 str++;
 318
 319                         if (*str == 0)          /* trailing op */
 320                                 return -EINVAL;
 321                 }
 322
 323                 /* find token length */
 324                 for (n = 0; str[n] != 0 && !isspace(str[n]); n++);
 325
 326                 /* match token */
 327                 if (libcfs_debug_token2mask(&t, str, n, is_subsys) != 0)
 328                         return -EINVAL;
 329
 330                 matched = 1;
 331                 if (op == '-')
 332                         m &= ~t;
 333                 else
 334                         m |= t;
 335
 336                 str += n;
 337         }
 338
 339         if (!matched)
 340                 return -EINVAL;
 341
 342         *mask = m;
 343         return 0;
 344 }
 345
 346 void libcfs_debug_dumplog_internal(void *arg)
 347 {
 348         CFS_DECL_JOURNAL_DATA;
 349
 350         CFS_PUSH_JOURNAL;
 351
 352         snprintf(debug_file_name, sizeof(debug_file_path) - 1, "%s.%ld.%ld",
 353                  debug_file_path, cfs_time_current_sec(), (long)arg);
 354         printk(KERN_ALERT "LustreError: dumping log to %s\n", debug_file_name);
 355         tracefile_dump_all_pages(debug_file_name);
 356
 357         CFS_POP_JOURNAL;
 358 }
 359
 360 int libcfs_debug_dumplog_thread(void *arg)
 361 {
 362         cfs_daemonize("");
 363         libcfs_debug_dumplog_internal(arg);
 364         cfs_waitq_signal(&debug_ctlwq);
 365         return 0;
 366 }
 367
 368 void libcfs_debug_dumplog(void)
 369 {
 370         int            rc;
 371         cfs_waitlink_t wait;
 372         ENTRY;
 373
 374         /* we're being careful to ensure that the kernel thread is
 375          * able to set our state to running as it exits before we
 376          * get to schedule() */
 377         cfs_waitlink_init(&wait);
 378         set_current_state(TASK_INTERRUPTIBLE);
 379         cfs_waitq_add(&debug_ctlwq, &wait);
 380
 381         rc = cfs_kernel_thread(libcfs_debug_dumplog_thread,
 382                                (void *)(long)cfs_curproc_pid(),
 383                                CLONE_VM | CLONE_FS | CLONE_FILES);
 384         if (rc < 0)
 385                 printk(KERN_ERR "LustreError: cannot start log dump thread: "
 386                        "%d\n", rc);
 387         else
 388                 cfs_waitq_wait(&wait, CFS_TASK_INTERRUPTIBLE);
 389
 390         /* be sure to teardown if kernel_thread() failed */
 391         cfs_waitq_del(&debug_ctlwq, &wait);
 392         set_current_state(TASK_RUNNING);
 393 }
 394
 395 int libcfs_debug_init(unsigned long bufsize)
 396 {
 397         int    rc;
 398
 399         cfs_waitq_init(&debug_ctlwq);
 400         rc = tracefile_init();
 401
 402         if (rc == 0)
 403                 libcfs_register_panic_notifier();
 404
 405         return rc;
 406 }
 407
 408 int libcfs_debug_cleanup(void)
 409 {
 410         libcfs_unregister_panic_notifier();
 411         tracefile_exit();
 412         return 0;
 413 }
 414
 415 int libcfs_debug_clear_buffer(void)
 416 {
 417         trace_flush_pages();
 418         return 0;
 419 }
 420
 421 /* Debug markers, although printed by S_LNET
 422  * should not be be marked as such. */
 423 #undef DEBUG_SUBSYSTEM
 424 #define DEBUG_SUBSYSTEM S_UNDEFINED
 425 int libcfs_debug_mark_buffer(char *text)
 426 {
 427         CDEBUG(D_TRACE,"***************************************************\n");
 428         CDEBUG(D_WARNING, "DEBUG MARKER: %s\n", text);
 429         CDEBUG(D_TRACE,"***************************************************\n");
 430
 431         return 0;
 432 }
 433 #undef DEBUG_SUBSYSTEM
 434 #define DEBUG_SUBSYSTEM S_LNET
 435
 436 void libcfs_debug_set_level(unsigned int debug_level)
 437 {
 438         printk(KERN_WARNING "Lustre: Setting portals debug level to %08x\n",
 439                debug_level);
 440         libcfs_debug = debug_level;
 441 }
 442
 443 EXPORT_SYMBOL(libcfs_debug_dumplog);
 444 EXPORT_SYMBOL(libcfs_debug_set_level);
 445
 446
 447 #else /* !__KERNEL__ */
 448
 449 #include <libcfs/libcfs.h>
 450
 451 #ifdef HAVE_SYS_USER_H
 452 # include <sys/user.h>
 453 #endif
 454
 455 #ifdef HAVE_CATAMOUNT_DATA_H
 456 #include <catamount/data.h>
 457 #include <catamount/lputs.h>
 458
 459 static char source_nid[16];
 460 /* 0 indicates no messages to console, 1 is errors, > 1 is all debug messages */
 461 static int toconsole = 1;
 462 unsigned int libcfs_console_ratelimit = 1;
 463 #else /* !HAVE_CATAMOUNT_DATA_H */
 464 #ifdef HAVE_NETDB_H
 465 #include <sys/utsname.h>
 466 #endif /* HAVE_CATAMOUNT_DATA_H */
 467 struct utsname *tmp_utsname;
 468 static char source_nid[sizeof(tmp_utsname->nodename)];
 469 #endif /* __KERNEL__ */
 470
 471 static int source_pid;
 472 int smp_processor_id = 1;
 473 char debug_file_path[1024];
 474 FILE *debug_file_fd;
 475
 476 int portals_do_debug_dumplog(void *arg)
 477 {
 478         printf("Look in %s\n", debug_file_name);
 479         return 0;
 480 }
 481
 482
 483 void portals_debug_print(void)
 484 {
 485         return;
 486 }
 487
 488
 489 void libcfs_debug_dumplog(void)
 490 {
 491         printf("Look in %s\n", debug_file_name);
 492         return;
 493 }
 494
 495 int libcfs_debug_init(unsigned long bufsize)
 496 {
 497         char *debug_mask = NULL;
 498         char *debug_subsys = NULL;
 499         char *debug_filename;
 500
 501 #ifdef HAVE_CATAMOUNT_DATA_H
 502         char *debug_console = NULL;
 503         char *debug_ratelimit = NULL;
 504
 505         snprintf(source_nid, sizeof(source_nid) - 1, "%u", _my_pnid);
 506         source_pid = _my_pid;
 507
 508         debug_console = getenv("LIBLUSTRE_DEBUG_CONSOLE");
 509         if (debug_console != NULL) {
 510                 toconsole = strtoul(debug_console, NULL, 0);
 511                 CDEBUG(D_INFO, "set liblustre toconsole to %u\n", toconsole);
 512         }
 513         debug_ratelimit = getenv("LIBLUSTRE_DEBUG_CONSOLE_RATELIMIT");
 514         if (debug_ratelimit != NULL) {
 515                 libcfs_console_ratelimit = strtoul(debug_ratelimit, NULL, 0);
 516                 CDEBUG(D_INFO, "set liblustre console ratelimit to %u\n", libcfs_console_ratelimit);
 517         }
 518 #else
 519         struct utsname myname;
 520
 521         if (uname(&myname) == 0)
 522                 strcpy(source_nid, myname.nodename);
 523         source_pid = getpid();
 524 #endif
 525         /* debug masks */
 526         debug_mask = getenv("LIBLUSTRE_DEBUG_MASK");
 527         if (debug_mask)
 528                 libcfs_debug = (unsigned int) strtol(debug_mask, NULL, 0);
 529
 530         debug_subsys = getenv("LIBLUSTRE_DEBUG_SUBSYS");
 531         if (debug_subsys)
 532                 libcfs_subsystem_debug =
 533                                 (unsigned int) strtol(debug_subsys, NULL, 0);
 534
 535         debug_filename = getenv("LIBLUSTRE_DEBUG_BASE");
 536         if (debug_filename)
 537                 strncpy(debug_file_path,debug_filename,sizeof(debug_file_path));
 538
 539         debug_filename = getenv("LIBLUSTRE_DEBUG_FILE");
 540         if (debug_filename)
 541                 strncpy(debug_file_name,debug_filename,sizeof(debug_file_path));
 542
 543         if (debug_file_name[0] == '\0' && debug_file_path[0] != '\0')
 544                 snprintf(debug_file_name, sizeof(debug_file_name) - 1,
 545                          "%s-%s-%lu.log", debug_file_path, source_nid, time(0));
 546
 547         if (strcmp(debug_file_name, "stdout") == 0 ||
 548             strcmp(debug_file_name, "-") == 0) {
 549                 debug_file_fd = stdout;
 550         } else if (strcmp(debug_file_name, "stderr") == 0) {
 551                 debug_file_fd = stderr;
 552         } else if (debug_file_name[0] != '\0') {
 553                 debug_file_fd = fopen(debug_file_name, "w");
 554                 if (debug_file_fd == NULL)
 555                         fprintf(stderr, "%s: unable to open '%s': %s\n",
 556                                 source_nid, debug_file_name, strerror(errno));
 557         }
 558
 559         if (debug_file_fd == NULL)
 560                 debug_file_fd = stdout;
 561
 562         return 0;
 563 }
 564
 565 int libcfs_debug_cleanup(void)
 566 {
 567         if (debug_file_fd != stdout && debug_file_fd != stderr)
 568                 fclose(debug_file_fd);
 569         return 0;
 570 }
 571
 572 int libcfs_debug_clear_buffer(void)
 573 {
 574         return 0;
 575 }
 576
 577 int libcfs_debug_mark_buffer(char *text)
 578 {
 579
 580         fprintf(debug_file_fd, "*******************************************************************************\n");
 581         fprintf(debug_file_fd, "DEBUG MARKER: %s\n", text);
 582         fprintf(debug_file_fd, "*******************************************************************************\n");
 583
 584         return 0;
 585 }
 586
 587 #ifdef HAVE_CATAMOUNT_DATA_H
 588 #define CATAMOUNT_MAXLINE (256-4)
 589 void catamount_printline(char *buf, size_t size)
 590 {
 591     char *pos = buf;
 592     int prsize = size;
 593
 594     while (prsize > 0){
 595         lputs(pos);
 596         pos += CATAMOUNT_MAXLINE;
 597         prsize -= CATAMOUNT_MAXLINE;
 598     }
 599 }
 600 #endif
 601
 602 int
 603 libcfs_debug_vmsg2(cfs_debug_limit_state_t *cdls,
 604                    int subsys, int mask,
 605                    const char *file, const char *fn, const int line,
 606                    const char *format1, va_list args,
 607                    const char *format2, ...)
 608 {
 609         struct timeval tv;
 610         int            nob;
 611         int            remain;
 612         va_list        ap;
 613         char           buf[PAGE_SIZE]; /* size 4096 used for compatimble with linux,
 614                                         * where message can`t be exceed PAGE_SIZE */
 615         int            console = 0;
 616         char *prefix = "Lustre";
 617
 618 #ifdef HAVE_CATAMOUNT_DATA_H
 619         /* toconsole == 0 - all messages to debug_file_fd
 620          * toconsole == 1 - warnings to console, all to debug_file_fd
 621          * toconsole >  1 - all debug to console */
 622         if ( ((mask & D_CANTMASK) &&
 623              (toconsole == 1)) || (toconsole > 1)) {
 624                 console = 1;
 625         }
 626 #endif
 627
 628         if ((!console) && (!debug_file_fd)) {
 629                 return 0;
 630         }
 631
 632         if (mask & (D_EMERG | D_ERROR))
 633                prefix = "LustreError";
 634
 635         nob = snprintf(buf, sizeof(buf), "%s: %u-%s:(%s:%d:%s()): ", prefix,
 636                        source_pid, source_nid, file, line, fn);
 637
 638         remain = sizeof(buf) - nob;
 639         if (format1) {
 640                 nob += vsnprintf(&buf[nob], remain, format1, args);
 641         }
 642
 643         remain = sizeof(buf) - nob;
 644         if ((format2) && (remain > 0)) {
 645                 va_start(ap, format2);
 646                 nob += vsnprintf(&buf[nob], remain, format2, ap);
 647                 va_end(ap);
 648         }
 649
 650 #ifdef HAVE_CATAMOUNT_DATA_H
 651         if (console) {
 652                 /* check rate limit for console */
 653                 if (cdls != NULL) {
 654                         cfs_time_t t = cdls->cdls_next +
 655                                        cfs_time_seconds(CDEBUG_MAX_LIMIT + 10);
 656                         cfs_duration_t  dmax = cfs_time_seconds(CDEBUG_MAX_LIMIT);
 657
 658                         if (libcfs_console_ratelimit &&
 659                                 cdls->cdls_next != 0 &&     /* not first time ever */
 660                                 !cfs_time_after(cfs_time_current(), cdls->cdls_next)) {
 661
 662                                 /* skipping a console message */
 663                                 cdls->cdls_count++;
 664                                 goto out_file;
 665                         }
 666
 667                         if (cfs_time_after(cfs_time_current(), t)) {
 668                                 /* last timeout was a long time ago */
 669                                 cdls->cdls_delay /= 8;
 670                         } else {
 671                                 cdls->cdls_delay *= 2;
 672
 673                                 if (cdls->cdls_delay < CFS_TICK)
 674                                         cdls->cdls_delay = CFS_TICK;
 675                                 else if (cdls->cdls_delay > dmax)
 676                                         cdls->cdls_delay = dmax;
 677                         }
 678
 679                         /* ensure cdls_next is never zero after it's been seen */
 680                         cdls->cdls_next = (cfs_time_current() + cdls->cdls_delay) | 1;
 681                 }
 682
 683                 if (cdls != NULL && cdls->cdls_count != 0) {
 684                         char buf2[100];
 685
 686                         nob = snprintf(buf2, sizeof(buf2),
 687                                        "Skipped %d previous similar message%s\n",
 688                                        cdls->cdls_count, (cdls->cdls_count > 1) ? "s" : "");
 689
 690                         catamount_printline(buf2, nob);
 691                         cdls->cdls_count = 0;
 692                         goto out_file;
 693                 }
 694                 catamount_printline(buf, nob);
 695        }
 696 out_file:
 697         /* return on toconsole > 1, as we don't want the user getting
 698         * spammed by the debug data */
 699         if (toconsole > 1)
 700                 return 0;
 701 #endif
 702         if (debug_file_fd == NULL)
 703                 return 0;
 704
 705         gettimeofday(&tv, NULL);
 706
 707         fprintf(debug_file_fd, "%lu.%06lu:%u:%s:(%s:%d:%s()): %s",
 708                 tv.tv_sec, tv.tv_usec, source_pid, source_nid,
 709                 file, line, fn, buf);
 710
 711         return 0;
 712 }
 713
 714 void
 715 libcfs_assertion_failed(const char *expr, const char *file, const char *func,
 716                         const int line)
 717 {
 718         libcfs_debug_msg(NULL, 0, D_EMERG, file, func, line,
 719                          "ASSERTION(%s) failed\n", expr);
 720         abort();
 721 }
 722
 723 #endif /* __KERNEL__ */