lnet/utils/debug.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2011, 2014, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/utils/debug.c
  37  * Some day I'll split all of this functionality into a cfs_debug module
  38  * of its own. That day is not today.
  39  */
  40
  41 #define __USE_FILE_OFFSET64
  42 #ifndef _GNU_SOURCE
  43 #define _GNU_SOURCE
  44 #endif
  45
  46 #include <unistd.h>
  47 #include <libcfs/libcfs.h>
  48 #include <libcfs/util/ioctl.h>
  49 #include <lnet/lnetctl.h>
  50
  51 static char rawbuf[8192];
  52 static char *buf = rawbuf;
  53 static int max = 8192;
  54 /*static int g_pfd = -1;*/
  55 static int subsystem_mask = ~0;
  56 static int debug_mask = ~0;
  57
  58 #define MAX_MARK_SIZE 256
  59
  60 static const char *libcfs_debug_subsystems[] = LIBCFS_DEBUG_SUBSYS_NAMES;
  61 static const char *libcfs_debug_masks[] = LIBCFS_DEBUG_MASKS_NAMES;
  62
  63 #define DAEMON_CTL_NAME         "/proc/sys/lnet/daemon_file"
  64 #define SUBSYS_DEBUG_CTL_NAME   "/proc/sys/lnet/subsystem_debug"
  65 #define DEBUG_CTL_NAME          "/proc/sys/lnet/debug"
  66 #define DUMP_KERNEL_CTL_NAME    "/proc/sys/lnet/dump_kernel"
  67
  68 static int
  69 dbg_open_ctlhandle(const char *str)
  70 {
  71         int fd;
  72         fd = open(str, O_WRONLY);
  73         if (fd < 0) {
  74                 fprintf(stderr, "open %s failed: %s\n", str,
  75                         strerror(errno));
  76                 return -1;
  77         }
  78         return fd;
  79 }
  80
  81 static void
  82 dbg_close_ctlhandle(int fd)
  83 {
  84         close(fd);
  85 }
  86
  87 static int
  88 dbg_write_cmd(int fd, char *str, int len)
  89 {
  90         int rc = write(fd, str, len);
  91
  92         return (rc == len ? 0 : 1);
  93 }
  94
  95
  96 static int do_debug_mask(char *name, int enable)
  97 {
  98         int found = 0;
  99         int i;
 100
 101         for (i = 0; libcfs_debug_subsystems[i] != NULL; i++) {
 102                 if (strcasecmp(name, libcfs_debug_subsystems[i]) == 0 ||
 103                     strcasecmp(name, "all_subs") == 0) {
 104                         printf("%s output from subsystem \"%s\"\n",
 105                                 enable ? "Enabling" : "Disabling",
 106                                 libcfs_debug_subsystems[i]);
 107                         if (enable)
 108                                 subsystem_mask |= (1 << i);
 109                         else
 110                                 subsystem_mask &= ~(1 << i);
 111                         found = 1;
 112                 }
 113         }
 114         for (i = 0; libcfs_debug_masks[i] != NULL; i++) {
 115                 if (strcasecmp(name, libcfs_debug_masks[i]) == 0 ||
 116                     strcasecmp(name, "all_types") == 0) {
 117                         printf("%s output of type \"%s\"\n",
 118                                 enable ? "Enabling" : "Disabling",
 119                                 libcfs_debug_masks[i]);
 120                         if (enable)
 121                                 debug_mask |= (1 << i);
 122                         else
 123                                 debug_mask &= ~(1 << i);
 124                         found = 1;
 125                 }
 126         }
 127
 128         return found;
 129 }
 130
 131 int dbg_initialize(int argc, char **argv)
 132 {
 133         return 0;
 134 }
 135
 136 int jt_dbg_filter(int argc, char **argv)
 137 {
 138         int i;
 139
 140         if (argc < 2) {
 141                 fprintf(stderr, "usage: %s <subsystem ID or debug mask>\n",
 142                         argv[0]);
 143                 return 0;
 144         }
 145
 146         for (i = 1; i < argc; i++)
 147                 if (!do_debug_mask(argv[i], 0))
 148                         fprintf(stderr, "Unknown subsystem or debug type: %s\n",
 149                                 argv[i]);
 150         return 0;
 151 }
 152
 153 int jt_dbg_show(int argc, char **argv)
 154 {
 155         int i;
 156
 157         if (argc < 2) {
 158                 fprintf(stderr, "usage: %s <subsystem ID or debug mask>\n",
 159                         argv[0]);
 160                 return 0;
 161         }
 162
 163         for (i = 1; i < argc; i++)
 164                 if (!do_debug_mask(argv[i], 1))
 165                         fprintf(stderr, "Unknown subsystem or debug type: %s\n",
 166                                 argv[i]);
 167
 168         return 0;
 169 }
 170
 171 static int applymask(char* procpath, int value)
 172 {
 173         int     rc;
 174         char    buf[64];
 175         int     len = snprintf(buf, 64, "%d", value);
 176
 177         int fd = dbg_open_ctlhandle(procpath);
 178         if (fd == -1) {
 179                 fprintf(stderr, "Unable to open %s: %s\n",
 180                         procpath, strerror(errno));
 181                 return fd;
 182         }
 183         rc = dbg_write_cmd(fd, buf, len+1);
 184         if (rc != 0) {
 185                 fprintf(stderr, "Write to %s failed: %s\n",
 186                         procpath, strerror(errno));
 187         }
 188
 189         dbg_close_ctlhandle(fd);
 190
 191         return rc;
 192 }
 193
 194 static void applymask_all(unsigned int subs_mask, unsigned int debug_mask)
 195 {
 196         applymask(SUBSYS_DEBUG_CTL_NAME, subs_mask);
 197         applymask(DEBUG_CTL_NAME, debug_mask);
 198         printf("Applied subsystem_debug=%d, debug=%d to /proc/sys/lnet\n",
 199                subs_mask, debug_mask);
 200 }
 201
 202 int jt_dbg_list(int argc, char **argv)
 203 {
 204         int i;
 205
 206         if (argc != 2) {
 207                 fprintf(stderr, "usage: %s <subs || types>\n", argv[0]);
 208                 return 0;
 209         }
 210
 211         if (strcasecmp(argv[1], "subs") == 0) {
 212                 printf("Subsystems: all_subs");
 213                 for (i = 0; libcfs_debug_subsystems[i] != NULL; i++)
 214                         if (libcfs_debug_subsystems[i][0])
 215                                 printf(", %s", libcfs_debug_subsystems[i]);
 216                 printf("\n");
 217         } else if (strcasecmp(argv[1], "types") == 0) {
 218                 printf("Types: all_types");
 219                 for (i = 0; libcfs_debug_masks[i] != NULL; i++)
 220                         printf(", %s", libcfs_debug_masks[i]);
 221                 printf("\n");
 222         } else if (strcasecmp(argv[1], "applymasks") == 0) {
 223                 applymask_all(subsystem_mask, debug_mask);
 224         }
 225         return 0;
 226 }
 227
 228 /* all strings nul-terminated; only the struct and hdr need to be freed */
 229 struct dbg_line {
 230         struct ptldebug_header *hdr;
 231         char                   *file;
 232         char                   *fn;
 233         char                   *text;
 234 };
 235
 236 static int cmp_rec(const void *p1, const void *p2)
 237 {
 238         struct dbg_line *d1 = *(struct dbg_line **)p1;
 239         struct dbg_line *d2 = *(struct dbg_line **)p2;
 240
 241         if (d1->hdr->ph_sec < d2->hdr->ph_sec)
 242                 return -1;
 243         if (d1->hdr->ph_sec == d2->hdr->ph_sec &&
 244             d1->hdr->ph_usec < d2->hdr->ph_usec)
 245                 return -1;
 246         if (d1->hdr->ph_sec == d2->hdr->ph_sec &&
 247             d1->hdr->ph_usec == d2->hdr->ph_usec)
 248                 return 0;
 249         return 1;
 250 }
 251
 252 static void print_rec(struct dbg_line ***linevp, int used, int fdout)
 253 {
 254         struct dbg_line **linev = *linevp;
 255         int               i;
 256
 257         qsort(linev, used, sizeof(struct dbg_line *), cmp_rec);
 258         for (i = 0; i < used; i++) {
 259                 struct dbg_line         *line = linev[i];
 260                 struct ptldebug_header  *hdr = line->hdr;
 261                 char                     out[4097];
 262                 char                    *buf = out;
 263                 int                      bytes;
 264                 ssize_t                  bytes_written;
 265
 266                 bytes = snprintf(out, sizeof(out),
 267                                 "%08x:%08x:%u.%u%s:%u.%06llu:%u:%u:%u:"
 268                                 "(%s:%u:%s()) %s",
 269                                 hdr->ph_subsys, hdr->ph_mask,
 270                                 hdr->ph_cpu_id, hdr->ph_type,
 271                                 hdr->ph_flags & PH_FLAG_FIRST_RECORD ? "F" : "",
 272                                 hdr->ph_sec, (unsigned long long)hdr->ph_usec,
 273                                 hdr->ph_stack, hdr->ph_pid, hdr->ph_extern_pid,
 274                                 line->file, hdr->ph_line_num, line->fn,
 275                                 line->text);
 276                 while (bytes > 0) {
 277                         bytes_written = write(fdout, buf, bytes);
 278                         if (bytes_written <= 0)
 279                                 break;
 280                         bytes -= bytes_written;
 281                         buf += bytes_written;
 282                 }
 283                 free(line->hdr);
 284                 free(line);
 285         }
 286         free(linev);
 287         *linevp = NULL;
 288 }
 289
 290 static int add_rec(struct dbg_line *line, struct dbg_line ***linevp, int *lenp,
 291                    int used)
 292 {
 293         struct dbg_line **linev = *linevp;
 294
 295         if (used == *lenp) {
 296                 int nlen = *lenp + 4096;
 297                 int nsize = nlen * sizeof(struct dbg_line *);
 298
 299                 linev = realloc(*linevp, nsize);
 300                 if (!linev)
 301                         return -ENOMEM;
 302
 303                 *linevp = linev;
 304                 *lenp = nlen;
 305         }
 306         linev[used] = line;
 307
 308         return 0;
 309 }
 310
 311 static void dump_hdr(unsigned long long offset, struct ptldebug_header *hdr)
 312 {
 313         fprintf(stderr, "badly-formed record at offset = %llu\n", offset);
 314         fprintf(stderr, "  len = %u\n", hdr->ph_len);
 315         fprintf(stderr, "  flags = %x\n", hdr->ph_flags);
 316         fprintf(stderr, "  subsystem = %x\n", hdr->ph_subsys);
 317         fprintf(stderr, "  mask = %x\n", hdr->ph_mask);
 318         fprintf(stderr, "  cpu_id = %u\n", hdr->ph_cpu_id);
 319         fprintf(stderr, "  type = %u\n", hdr->ph_type);
 320         fprintf(stderr, "  seconds = %u\n", hdr->ph_sec);
 321         fprintf(stderr, "  microseconds = %lu\n", (long)hdr->ph_usec);
 322         fprintf(stderr, "  stack = %u\n", hdr->ph_stack);
 323         fprintf(stderr, "  pid = %u\n", hdr->ph_pid);
 324         fprintf(stderr, "  host pid = %u\n", hdr->ph_extern_pid);
 325         fprintf(stderr, "  line number = %u\n", hdr->ph_line_num);
 326 }
 327
 328 #define HDR_SIZE sizeof(*hdr)
 329
 330 static int parse_buffer(int fdin, int fdout)
 331 {
 332         struct dbg_line         *line;
 333         struct ptldebug_header  *hdr;
 334         char                     buf[4097];
 335         char                    *ptr;
 336         unsigned long            dropped = 0;
 337         unsigned long            kept = 0;
 338         unsigned long            bad = 0;
 339         struct dbg_line         **linev = NULL;
 340         int                      linev_len = 0;
 341         int                      rc;
 342
 343         hdr = (void *)buf;
 344
 345         while (1) {
 346                 int first_bad = 1;
 347                 int count;
 348
 349                 count = HDR_SIZE;
 350                 ptr = buf;
 351 readhdr:
 352                 rc = read(fdin, ptr, count);
 353                 if (rc <= 0)
 354                         goto print;
 355
 356                 ptr += rc;
 357                 count -= rc;
 358                 if (count > 0)
 359                         goto readhdr;
 360
 361                 if (hdr->ph_len > 4094 ||       /* is this header bogus? */
 362                     hdr->ph_stack > 65536 ||
 363                     hdr->ph_sec < (1 << 30) ||
 364                     hdr->ph_usec > 1000000000 ||
 365                     hdr->ph_line_num > 65536) {
 366                         if (first_bad)
 367                                 dump_hdr(lseek(fdin, 0, SEEK_CUR), hdr);
 368                         bad += first_bad;
 369                         first_bad = 0;
 370
 371                         /* try to restart on next line */
 372                         while (count < HDR_SIZE && buf[count] != '\n')
 373                                 count++;
 374                         if (buf[count] == '\n')
 375                                 count++; /* move past '\n' */
 376                         if (HDR_SIZE - count > 0) {
 377                                 int left = HDR_SIZE - count;
 378
 379                                 memmove(buf, buf + count, left);
 380                                 ptr = buf + left;
 381
 382                                 goto readhdr;
 383                         }
 384
 385                         continue;
 386                 }
 387
 388                 if (hdr->ph_len == 0)
 389                         continue;
 390
 391                 count = hdr->ph_len - HDR_SIZE;
 392 readmore:
 393                 rc = read(fdin, ptr, count);
 394                 if (rc <= 0)
 395                         break;
 396
 397                 ptr += rc;
 398                 count -= rc;
 399                 if (count > 0)
 400                         goto readmore;
 401
 402                 first_bad = 1;
 403
 404                 if ((hdr->ph_subsys && !(subsystem_mask & hdr->ph_subsys)) ||
 405                     (hdr->ph_mask && !(debug_mask & hdr->ph_mask))) {
 406                         dropped++;
 407                         continue;
 408                 }
 409
 410 retry_alloc:
 411                 line = malloc(sizeof(*line));
 412                 if (line == NULL) {
 413                         if (linev) {
 414                                 fprintf(stderr, "error: line malloc(%u): "
 415                                         "printing accumulated records\n",
 416                                         (unsigned int)sizeof(*line));
 417                                 print_rec(&linev, kept, fdout);
 418
 419                                 goto retry_alloc;
 420                         }
 421                         fprintf(stderr, "error: line malloc(%u): exiting\n",
 422                                 (unsigned int)sizeof(*line));
 423                         break;
 424                 }
 425
 426                 line->hdr = malloc(hdr->ph_len + 1);
 427                 if (line->hdr == NULL) {
 428                         free(line);
 429                         if (linev) {
 430                                 fprintf(stderr, "error: hdr malloc(%u): "
 431                                         "printing accumulated records\n",
 432                                         hdr->ph_len + 1);
 433                                 print_rec(&linev, kept, fdout);
 434
 435                                 goto retry_alloc;
 436                         }
 437                         fprintf(stderr, "error: hdr malloc(%u): exiting\n",
 438                                         hdr->ph_len + 1);
 439                         break;
 440                 }
 441
 442                 ptr = (void *)line->hdr;
 443                 memcpy(line->hdr, buf, hdr->ph_len);
 444                 ptr[hdr->ph_len] = '\0';
 445
 446                 ptr += sizeof(*hdr);
 447                 line->file = ptr;
 448                 ptr += strlen(line->file) + 1;
 449                 line->fn = ptr;
 450                 ptr += strlen(line->fn) + 1;
 451                 line->text = ptr;
 452
 453 retry_add:
 454                 if (add_rec(line, &linev, &linev_len, kept) < 0) {
 455                         if (linev) {
 456                                 fprintf(stderr, "error: add_rec[%u] failed; "
 457                                         "print accumulated records\n",
 458                                         linev_len);
 459                                 print_rec(&linev, kept, fdout);
 460
 461                                 goto retry_add;
 462                         }
 463                         fprintf(stderr, "error: add_rec[0] failed; exiting\n");
 464                         break;
 465                 }
 466                 kept++;
 467         }
 468
 469 print:
 470         if (linev)
 471                 print_rec(&linev, kept, fdout);
 472
 473         printf("Debug log: %lu lines, %lu kept, %lu dropped, %lu bad.\n",
 474                 dropped + kept + bad, kept, dropped, bad);
 475
 476         return 0;
 477 }
 478
 479 int jt_dbg_debug_kernel(int argc, char **argv)
 480 {
 481         struct stat     st;
 482         char            filename[PATH_MAX];
 483         int             raw = 0;
 484         int             save_errno;
 485         int             fdin;
 486         int             fdout;
 487         int             rc;
 488
 489         if (argc > 3) {
 490                 fprintf(stderr, "usage: %s [file] [raw]\n", argv[0]);
 491                 return 0;
 492         }
 493
 494         if (argc > 2) {
 495                 raw = atoi(argv[2]);
 496         } else if (argc > 1 && (argv[1][0] == '0' || argv[1][0] == '1')) {
 497                 raw = atoi(argv[1]);
 498                 argc--;
 499         }
 500
 501         /* If we are dumping raw (which means no conversion step to ASCII)
 502          * then dump directly to any supplied filename, otherwise this is
 503          * just a temp file and we dump to the real file at convert time. */
 504         if (argc > 1 && raw) {
 505                 if (strlen(argv[1]) >= sizeof(filename)) {
 506                         fprintf(stderr, "File name too long: %s\n", argv[1]);
 507                         return 1;
 508                 }
 509                 strncpy(filename, argv[1], sizeof(filename));
 510         } else {
 511                 if (snprintf(filename, sizeof(filename), "%s"CFS_TIME_T".%u",
 512                              LIBCFS_DEBUG_FILE_PATH_DEFAULT, time(NULL),
 513                              getpid())
 514                     >= sizeof(filename)) {
 515                         fprintf(stderr, "File name too long\n");
 516                         return 1;
 517                 }
 518         }
 519
 520         if (stat(filename, &st) == 0 && S_ISREG(st.st_mode))
 521                 unlink(filename);
 522
 523         fdin = dbg_open_ctlhandle(DUMP_KERNEL_CTL_NAME);
 524         if (fdin < 0) {
 525                 fprintf(stderr, "open(dump_kernel) failed: %s\n",
 526                         strerror(errno));
 527                 return 1;
 528         }
 529
 530         rc = dbg_write_cmd(fdin, filename, strlen(filename));
 531         save_errno = errno;
 532         dbg_close_ctlhandle(fdin);
 533         if (rc != 0) {
 534                 fprintf(stderr, "write(%s) failed: %s\n", filename,
 535                         strerror(save_errno));
 536                 return 1;
 537         }
 538
 539         if (raw)
 540                 return 0;
 541
 542         fdin = open(filename, O_RDONLY);
 543         if (fdin < 0) {
 544                 if (errno == ENOENT) /* no dump file created */
 545                         return 0;
 546                 fprintf(stderr, "fopen(%s) failed: %s\n", filename,
 547                         strerror(errno));
 548                 return 1;
 549         }
 550         if (argc > 1) {
 551                 fdout = open(argv[1], O_WRONLY | O_CREAT | O_TRUNC,
 552                              S_IRUSR | S_IWUSR);
 553                 if (fdout < 0) {
 554                         fprintf(stderr, "fopen(%s) failed: %s\n", argv[1],
 555                                 strerror(errno));
 556                         close(fdin);
 557                         return 1;
 558                 }
 559         } else {
 560                 fdout = fileno(stdout);
 561         }
 562
 563         rc = parse_buffer(fdin, fdout);
 564         close(fdin);
 565         if (argc > 1)
 566                 close(fdout);
 567         if (rc) {
 568                 fprintf(stderr, "parse_buffer failed; leaving tmp file %s "
 569                         "behind.\n", filename);
 570         } else {
 571                 rc = unlink(filename);
 572                 if (rc)
 573                         fprintf(stderr, "dumped successfully, but couldn't "
 574                                 "unlink tmp file %s: %s\n", filename,
 575                                 strerror(errno));
 576         }
 577
 578         return rc;
 579 }
 580
 581 int jt_dbg_debug_file(int argc, char **argv)
 582 {
 583         int fdin;
 584         int fdout;
 585         int rc;
 586
 587         if (argc > 3 || argc < 2) {
 588                 fprintf(stderr, "usage: %s <input> [output]\n", argv[0]);
 589                 return 0;
 590         }
 591
 592         fdin = open(argv[1], O_RDONLY | O_LARGEFILE);
 593         if (fdin < 0) {
 594                 fprintf(stderr, "open(%s) failed: %s\n", argv[1],
 595                         strerror(errno));
 596                 return 1;
 597         }
 598         if (argc > 2) {
 599                 fdout = open(argv[2],
 600                              O_CREAT | O_TRUNC | O_WRONLY | O_LARGEFILE,
 601                              0600);
 602                 if (fdout < 0) {
 603                         fprintf(stderr, "open(%s) failed: %s\n", argv[2],
 604                                 strerror(errno));
 605                         close(fdin);
 606                         return 1;
 607                 }
 608         } else {
 609                 fdout = fileno(stdout);
 610         }
 611
 612         rc = parse_buffer(fdin, fdout);
 613
 614         close(fdin);
 615         if (fdout != fileno(stdout))
 616                 close(fdout);
 617
 618         return rc;
 619 }
 620
 621 const char debug_daemon_usage[] = "usage: %s {start file [MB]|stop}\n";
 622
 623 int jt_dbg_debug_daemon(int argc, char **argv)
 624 {
 625         int rc;
 626         int fd;
 627
 628         if (argc <= 1) {
 629                 fprintf(stderr, debug_daemon_usage, argv[0]);
 630                 return 1;
 631         }
 632
 633         fd = dbg_open_ctlhandle(DAEMON_CTL_NAME);
 634         if (fd < 0)
 635                 return -1;
 636
 637         rc = -1;
 638         if (strcasecmp(argv[1], "start") == 0) {
 639                 if (argc < 3 || argc > 4 ||
 640                     (argc == 4 && strlen(argv[3]) > 5)) {
 641                         fprintf(stderr, debug_daemon_usage, argv[0]);
 642                         goto out;
 643                 }
 644                 if (argc == 4) {
 645                         char             buf[12];
 646                         const long       min_size = 10;
 647                         const long       max_size = 20480;
 648                         long             size;
 649                         char            *end;
 650
 651                         size = strtoul(argv[3], &end, 0);
 652                         if (size < min_size ||
 653                             size > max_size ||
 654                             *end != 0) {
 655                                 fprintf(stderr, "size %s invalid, must be in "
 656                                         "the range %ld-%ld MB\n", argv[3],
 657                                         min_size, max_size);
 658                                 goto out;
 659                         }
 660                         snprintf(buf, sizeof(buf), "size=%ld", size);
 661                         rc = dbg_write_cmd(fd, buf, strlen(buf));
 662
 663                         if (rc != 0) {
 664                                 fprintf(stderr, "set %s failed: %s\n",
 665                                         buf, strerror(errno));
 666                                 goto out;
 667                         }
 668                 }
 669
 670                 rc = dbg_write_cmd(fd, argv[2], strlen(argv[2]));
 671                 if (rc != 0) {
 672                         fprintf(stderr, "start debug_daemon on %s failed: %s\n",
 673                                 argv[2], strerror(errno));
 674                         goto out;
 675                 }
 676                 rc = 0;
 677                 goto out;
 678         }
 679         if (strcasecmp(argv[1], "stop") == 0) {
 680                 rc = dbg_write_cmd(fd, "stop", 4);
 681                 if (rc != 0) {
 682                         fprintf(stderr, "stopping debug_daemon failed: %s\n",
 683                                 strerror(errno));
 684                         goto out;
 685                 }
 686
 687                 rc = 0;
 688                 goto out;
 689         }
 690
 691         fprintf(stderr, debug_daemon_usage, argv[0]);
 692         rc = -1;
 693 out:
 694         dbg_close_ctlhandle(fd);
 695         return rc;
 696 }
 697
 698 int jt_dbg_clear_debug_buf(int argc, char **argv)
 699 {
 700         int                      rc;
 701         struct libcfs_ioctl_data data;
 702
 703         if (argc != 1) {
 704                 fprintf(stderr, "usage: %s\n", argv[0]);
 705                 return 0;
 706         }
 707
 708         memset(&data, 0, sizeof(data));
 709         if (libcfs_ioctl_pack(&data, &buf, max) != 0) {
 710                 fprintf(stderr, "libcfs_ioctl_pack failed.\n");
 711                 return -1;
 712         }
 713
 714         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_CLEAR_DEBUG, buf);
 715         if (rc) {
 716                 fprintf(stderr, "IOC_LIBCFS_CLEAR_DEBUG failed: %s\n",
 717                         strerror(errno));
 718                 return -1;
 719         }
 720         return 0;
 721 }
 722
 723 int jt_dbg_mark_debug_buf(int argc, char **argv)
 724 {
 725         static char              scratch[MAX_MARK_SIZE] = "";
 726         struct libcfs_ioctl_data data;
 727         char                    *text;
 728         int                      rc;
 729
 730         memset(&data, 0, sizeof(data));
 731
 732         if (argc > 1) {
 733                 int count, max_size = sizeof(scratch) - 1;
 734
 735                 strncpy(scratch, argv[1], max_size);
 736                 max_size -= strlen(argv[1]);
 737                 for (count = 2; (count < argc) && (max_size > 1); count++) {
 738                         strncat(scratch, " ", max_size);
 739                         max_size -= 1;
 740                         strncat(scratch, argv[count], max_size);
 741                         max_size -= strlen(argv[count]);
 742                 }
 743                 scratch[sizeof(scratch) - 1] = '\0';
 744                 text = scratch;
 745         } else {
 746                 time_t now = time(NULL);
 747                 text = ctime(&now);
 748         }
 749
 750         data.ioc_inllen1 = strlen(text) + 1;
 751         data.ioc_inlbuf1 = text;
 752
 753         if (libcfs_ioctl_pack(&data, &buf, max) != 0) {
 754                 fprintf(stderr, "libcfs_ioctl_pack failed.\n");
 755                 return -1;
 756         }
 757
 758         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_MARK_DEBUG, buf);
 759         if (rc) {
 760                 fprintf(stderr, "IOC_LIBCFS_MARK_DEBUG failed: %s\n",
 761                         strerror(errno));
 762                 return -1;
 763         }
 764         return 0;
 765 }
 766
 767 static struct mod_paths {
 768         char *name, *path;
 769 } mod_paths[] = {
 770         { "libcfs", "libcfs/libcfs" },
 771         { "lnet", "lnet/lnet" },
 772         { "ko2iblnd", "lnet/klnds/o2iblnd" },
 773         { "kgnilnd", "lnet/klnds/gnilnd"},
 774         { "ksocklnd", "lnet/klnds/socklnd" },
 775         { "obdclass", "lustre/obdclass" },
 776         { "llog_test", "lustre/obdclass" },
 777         { "ptlrpc_gss", "lustre/ptlrpc/gss" },
 778         { "ptlrpc", "lustre/ptlrpc" },
 779         { "gks", "lustre/sec/gks" },
 780         { "gkc", "lustre/sec/gks" },
 781         { "ost", "lustre/ost" },
 782         { "osc", "lustre/osc" },
 783         { "mds", "lustre/mds" },
 784         { "mdc", "lustre/mdc" },
 785         { "lustre", "lustre/llite" },
 786         { "llite_lloop", "lustre/llite" },
 787         { "ldiskfs", "ldiskfs" },
 788         { "obdecho", "lustre/obdecho" },
 789         { "ldlm", "lustre/ldlm" },
 790         { "obdfilter", "lustre/obdfilter" },
 791         { "lov", "lustre/lov" },
 792         { "lmv", "lustre/lmv" },
 793         { "lquota", "lustre/quota" },
 794         { "mgs", "lustre/mgs" },
 795         { "mgc", "lustre/mgc" },
 796         { "mdt", "lustre/mdt" },
 797         { "mdd", "lustre/mdd" },
 798         { "osd", "lustre/osd" },
 799         { "cmm", "lustre/cmm" },
 800         {"fid", "lustre/fid"},
 801         {"fld", "lustre/fld"},
 802         {"lod", "lustre/lod"},
 803         {"osp", "lustre/osp"},
 804         { "lfsck", "lustre/lfsck" },
 805         {NULL, NULL}
 806 };
 807
 808 int jt_dbg_modules(int argc, char **argv)
 809 {
 810         struct mod_paths *mp;
 811         char             *path = "";
 812         const char       *proc = "/proc/modules";
 813         char              modname[128];
 814         char              buf[4096];
 815         unsigned long     modaddr;
 816         FILE             *file;
 817
 818         if (argc >= 2)
 819                 path = argv[1];
 820         if (argc > 3) {
 821                 printf("%s [path] [kernel]\n", argv[0]);
 822                 return 0;
 823         }
 824
 825         file = fopen(proc, "r");
 826         if (!file) {
 827                 printf("failed open %s: %s\n", proc, strerror(errno));
 828                 return 0;
 829         }
 830
 831         while (fgets(buf, sizeof(buf), file) != NULL) {
 832                 if (sscanf(buf, "%s %*s %*s %*s %*s %lx",
 833                            modname, &modaddr) == 2) {
 834                         for (mp = mod_paths; mp->name != NULL; mp++) {
 835                                 if (!strcmp(mp->name, modname))
 836                                         break;
 837                         }
 838                         if (mp->name) {
 839                                 printf("add-symbol-file %s%s%s/%s.o 0x%0lx\n",
 840                                         path, path[0] ? "/" : "",
 841                                         mp->path, mp->name, modaddr);
 842                         }
 843                 }
 844         }
 845
 846         fclose(file);
 847         return 0;
 848 }
 849
 850 int jt_dbg_panic(int argc, char **argv)
 851 {
 852         int                      rc;
 853         struct libcfs_ioctl_data data;
 854
 855         if (argc != 1) {
 856                 fprintf(stderr, "usage: %s\n", argv[0]);
 857                 return 0;
 858         }
 859
 860         memset(&data, 0, sizeof(data));
 861         if (libcfs_ioctl_pack(&data, &buf, max) != 0) {
 862                 fprintf(stderr, "libcfs_ioctl_pack failed.\n");
 863                 return -1;
 864         }
 865
 866         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_PANIC, buf);
 867         if (rc) {
 868                 fprintf(stderr, "IOC_LIBCFS_PANIC failed: %s\n",
 869                         strerror(errno));
 870                 return -1;
 871         }
 872         return 0;
 873 }