lnet/utils/debug.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2015, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lnet/utils/debug.c
  33  * Some day I'll split all of this functionality into a cfs_debug module
  34  * of its own. That day is not today.
  35  */
  36
  37 #define __USE_FILE_OFFSET64
  38 #ifndef _GNU_SOURCE
  39 #define _GNU_SOURCE
  40 #endif
  41
  42 #include <errno.h>
  43 #include <fcntl.h>
  44 #include <limits.h>
  45 #include <stdarg.h>
  46 #include <stdio.h>
  47 #include <stdlib.h>
  48 #include <string.h>
  49 #include <sys/ioctl.h>
  50 #include <sys/types.h>
  51 #include <sys/stat.h>
  52 #include <time.h>
  53 #include <unistd.h>
  54
  55 #include <linux/types.h>
  56
  57 #include <libcfs/util/ioctl.h>
  58 #include <libcfs/util/param.h>
  59 #include <libcfs/libcfs_debug.h>
  60 #include <lnet/lnetctl.h>
  61
  62 static char rawbuf[8192];
  63 static char *buf = rawbuf;
  64 static int max = 8192;
  65 /*static int g_pfd = -1;*/
  66 static int subsystem_mask = ~0;
  67 static int debug_mask = ~0;
  68
  69 #define MAX_MARK_SIZE 256
  70
  71 static const char *libcfs_debug_subsystems[] = LIBCFS_DEBUG_SUBSYS_NAMES;
  72 static const char *libcfs_debug_masks[] = LIBCFS_DEBUG_MASKS_NAMES;
  73
  74 #define DAEMON_CTL_NAME         "daemon_file"
  75 #define SUBSYS_DEBUG_CTL_NAME   "subsystem_debug"
  76 #define DEBUG_CTL_NAME          "debug"
  77 #define DUMP_KERNEL_CTL_NAME    "dump_kernel"
  78
  79 /*
  80  * Open the parameter file "debug" which controls the debugging
  81  * flags used to determine what information ends up in the lustre
  82  * logs collected by lctl dk or the debug daemon.
  83  */
  84 static int
  85 dbg_open_ctlhandle(const char *str)
  86 {
  87         glob_t path;
  88         int fd, rc;
  89
  90         rc = cfs_get_param_paths(&path, "%s", str);
  91         if (rc != 0) {
  92                 fprintf(stderr, "invalid parameter '%s'\n", str);
  93                 return -1;
  94         }
  95
  96         fd = open(path.gl_pathv[0], O_WRONLY);
  97         if (fd < 0)
  98                 fprintf(stderr, "open '%s' failed: %s\n",
  99                         path.gl_pathv[0], strerror(errno));
 100
 101         cfs_free_param_data(&path);
 102         return fd;
 103 }
 104
 105 static void
 106 dbg_close_ctlhandle(int fd)
 107 {
 108         close(fd);
 109 }
 110
 111 static int
 112 dbg_write_cmd(int fd, char *str, int len)
 113 {
 114         int rc = write(fd, str, len);
 115
 116         return (rc == len ? 0 : 1);
 117 }
 118
 119
 120 static int do_debug_mask(char *name, int enable)
 121 {
 122         int found = 0;
 123         int i;
 124
 125         for (i = 0; libcfs_debug_subsystems[i] != NULL; i++) {
 126                 if (strcasecmp(name, libcfs_debug_subsystems[i]) == 0 ||
 127                     strcasecmp(name, "all_subs") == 0) {
 128                         printf("%s output from subsystem \"%s\"\n",
 129                                 enable ? "Enabling" : "Disabling",
 130                                 libcfs_debug_subsystems[i]);
 131                         if (enable)
 132                                 subsystem_mask |= (1 << i);
 133                         else
 134                                 subsystem_mask &= ~(1 << i);
 135                         found = 1;
 136                 }
 137         }
 138         for (i = 0; libcfs_debug_masks[i] != NULL; i++) {
 139                 if (strcasecmp(name, libcfs_debug_masks[i]) == 0 ||
 140                     strcasecmp(name, "all_types") == 0) {
 141                         printf("%s output of type \"%s\"\n",
 142                                 enable ? "Enabling" : "Disabling",
 143                                 libcfs_debug_masks[i]);
 144                         if (enable)
 145                                 debug_mask |= (1 << i);
 146                         else
 147                                 debug_mask &= ~(1 << i);
 148                         found = 1;
 149                 }
 150         }
 151
 152         return found;
 153 }
 154
 155 int dbg_initialize(int argc, char **argv)
 156 {
 157         return 0;
 158 }
 159
 160 int jt_dbg_filter(int argc, char **argv)
 161 {
 162         int i;
 163
 164         if (argc < 2) {
 165                 fprintf(stderr, "usage: %s <subsystem ID or debug mask>\n",
 166                         argv[0]);
 167                 return 0;
 168         }
 169
 170         for (i = 1; i < argc; i++)
 171                 if (!do_debug_mask(argv[i], 0))
 172                         fprintf(stderr, "Unknown subsystem or debug type: %s\n",
 173                                 argv[i]);
 174         return 0;
 175 }
 176
 177 int jt_dbg_show(int argc, char **argv)
 178 {
 179         int i;
 180
 181         if (argc < 2) {
 182                 fprintf(stderr, "usage: %s <subsystem ID or debug mask>\n",
 183                         argv[0]);
 184                 return 0;
 185         }
 186
 187         for (i = 1; i < argc; i++)
 188                 if (!do_debug_mask(argv[i], 1))
 189                         fprintf(stderr, "Unknown subsystem or debug type: %s\n",
 190                                 argv[i]);
 191
 192         return 0;
 193 }
 194
 195 static int applymask(char *param, int value)
 196 {
 197         int     rc;
 198         char    buf[64];
 199         int     len = snprintf(buf, 64, "%d", value);
 200
 201         int fd = dbg_open_ctlhandle(param);
 202         if (fd < 0)
 203                 return fd;
 204
 205         rc = dbg_write_cmd(fd, buf, len+1);
 206         if (rc != 0) {
 207                 fprintf(stderr, "Write to %s failed: %s\n",
 208                         param, strerror(errno));
 209         }
 210
 211         dbg_close_ctlhandle(fd);
 212
 213         return rc;
 214 }
 215
 216 static void applymask_all(unsigned int subs_mask, unsigned int debug_mask)
 217 {
 218         applymask(SUBSYS_DEBUG_CTL_NAME, subs_mask);
 219         applymask(DEBUG_CTL_NAME, debug_mask);
 220         printf("Applied subsystem_debug=%d, debug=%d to lnet\n",
 221                subs_mask, debug_mask);
 222 }
 223
 224 int jt_dbg_list(int argc, char **argv)
 225 {
 226         int i;
 227
 228         if (argc != 2) {
 229                 fprintf(stderr, "usage: %s <subs || types>\n", argv[0]);
 230                 return 0;
 231         }
 232
 233         if (strcasecmp(argv[1], "subs") == 0) {
 234                 printf("Subsystems: all_subs");
 235                 for (i = 0; libcfs_debug_subsystems[i] != NULL; i++)
 236                         if (libcfs_debug_subsystems[i][0])
 237                                 printf(", %s", libcfs_debug_subsystems[i]);
 238                 printf("\n");
 239         } else if (strcasecmp(argv[1], "types") == 0) {
 240                 printf("Types: all_types");
 241                 for (i = 0; libcfs_debug_masks[i] != NULL; i++)
 242                         printf(", %s", libcfs_debug_masks[i]);
 243                 printf("\n");
 244         } else if (strcasecmp(argv[1], "applymasks") == 0) {
 245                 applymask_all(subsystem_mask, debug_mask);
 246         }
 247         return 0;
 248 }
 249
 250 /* all strings nul-terminated; only the struct and hdr need to be freed */
 251 struct dbg_line {
 252         struct ptldebug_header *hdr;
 253         char                   *file;
 254         char                   *fn;
 255         char                   *text;
 256 };
 257
 258 static int cmp_rec(const void *p1, const void *p2)
 259 {
 260         struct dbg_line *d1 = *(struct dbg_line **)p1;
 261         struct dbg_line *d2 = *(struct dbg_line **)p2;
 262
 263         if (d1->hdr->ph_sec < d2->hdr->ph_sec)
 264                 return -1;
 265         if (d1->hdr->ph_sec == d2->hdr->ph_sec &&
 266             d1->hdr->ph_usec < d2->hdr->ph_usec)
 267                 return -1;
 268         if (d1->hdr->ph_sec == d2->hdr->ph_sec &&
 269             d1->hdr->ph_usec == d2->hdr->ph_usec)
 270                 return 0;
 271         return 1;
 272 }
 273
 274 static void print_rec(struct dbg_line ***linevp, int used, int fdout)
 275 {
 276         struct dbg_line **linev = *linevp;
 277         int               i;
 278
 279         qsort(linev, used, sizeof(struct dbg_line *), cmp_rec);
 280         for (i = 0; i < used; i++) {
 281                 struct dbg_line         *line = linev[i];
 282                 struct ptldebug_header  *hdr = line->hdr;
 283                 char                     out[4097];
 284                 char                    *buf = out;
 285                 int                      bytes;
 286                 ssize_t                  bytes_written;
 287
 288                 bytes = snprintf(out, sizeof(out),
 289                                 "%08x:%08x:%u.%u%s:%u.%06llu:%u:%u:%u:"
 290                                 "(%s:%u:%s()) %s",
 291                                 hdr->ph_subsys, hdr->ph_mask,
 292                                 hdr->ph_cpu_id, hdr->ph_type,
 293                                 hdr->ph_flags & PH_FLAG_FIRST_RECORD ? "F" : "",
 294                                 hdr->ph_sec, (unsigned long long)hdr->ph_usec,
 295                                 hdr->ph_stack, hdr->ph_pid, hdr->ph_extern_pid,
 296                                 line->file, hdr->ph_line_num, line->fn,
 297                                 line->text);
 298                 while (bytes > 0) {
 299                         bytes_written = write(fdout, buf, bytes);
 300                         if (bytes_written <= 0)
 301                                 break;
 302                         bytes -= bytes_written;
 303                         buf += bytes_written;
 304                 }
 305                 free(line->hdr);
 306                 free(line);
 307         }
 308         free(linev);
 309         *linevp = NULL;
 310 }
 311
 312 static int add_rec(struct dbg_line *line, struct dbg_line ***linevp, int *lenp,
 313                    int used)
 314 {
 315         struct dbg_line **linev = *linevp;
 316
 317         if (used == *lenp) {
 318                 int nlen = *lenp + 4096;
 319                 int nsize = nlen * sizeof(struct dbg_line *);
 320
 321                 linev = realloc(*linevp, nsize);
 322                 if (!linev)
 323                         return -ENOMEM;
 324
 325                 *linevp = linev;
 326                 *lenp = nlen;
 327         }
 328         linev[used] = line;
 329
 330         return 0;
 331 }
 332
 333 static void dump_hdr(unsigned long long offset, struct ptldebug_header *hdr)
 334 {
 335         fprintf(stderr, "badly-formed record at offset = %llu\n", offset);
 336         fprintf(stderr, "  len = %u\n", hdr->ph_len);
 337         fprintf(stderr, "  flags = %x\n", hdr->ph_flags);
 338         fprintf(stderr, "  subsystem = %x\n", hdr->ph_subsys);
 339         fprintf(stderr, "  mask = %x\n", hdr->ph_mask);
 340         fprintf(stderr, "  cpu_id = %u\n", hdr->ph_cpu_id);
 341         fprintf(stderr, "  type = %u\n", hdr->ph_type);
 342         fprintf(stderr, "  seconds = %u\n", hdr->ph_sec);
 343         fprintf(stderr, "  microseconds = %lu\n", (long)hdr->ph_usec);
 344         fprintf(stderr, "  stack = %u\n", hdr->ph_stack);
 345         fprintf(stderr, "  pid = %u\n", hdr->ph_pid);
 346         fprintf(stderr, "  host pid = %u\n", hdr->ph_extern_pid);
 347         fprintf(stderr, "  line number = %u\n", hdr->ph_line_num);
 348 }
 349
 350 #define HDR_SIZE sizeof(*hdr)
 351
 352 static int parse_buffer(int fdin, int fdout)
 353 {
 354         struct dbg_line         *line;
 355         struct ptldebug_header  *hdr;
 356         char                     buf[4097];
 357         char                    *ptr;
 358         unsigned long            dropped = 0;
 359         unsigned long            kept = 0;
 360         unsigned long            bad = 0;
 361         struct dbg_line         **linev = NULL;
 362         int                      linev_len = 0;
 363         int                      rc;
 364
 365         hdr = (void *)buf;
 366
 367         while (1) {
 368                 int first_bad = 1;
 369                 int count;
 370
 371                 count = HDR_SIZE;
 372                 ptr = buf;
 373 readhdr:
 374                 rc = read(fdin, ptr, count);
 375                 if (rc <= 0)
 376                         goto print;
 377
 378                 ptr += rc;
 379                 count -= rc;
 380                 if (count > 0)
 381                         goto readhdr;
 382
 383                 if (hdr->ph_len > 4094 ||       /* is this header bogus? */
 384                     hdr->ph_stack > 65536 ||
 385                     hdr->ph_sec < (1 << 30) ||
 386                     hdr->ph_usec > 1000000000 ||
 387                     hdr->ph_line_num > 65536) {
 388                         if (first_bad)
 389                                 dump_hdr(lseek(fdin, 0, SEEK_CUR), hdr);
 390                         bad += first_bad;
 391                         first_bad = 0;
 392
 393                         /* try to restart on next line */
 394                         while (count < HDR_SIZE && buf[count] != '\n')
 395                                 count++;
 396                         if (buf[count] == '\n')
 397                                 count++; /* move past '\n' */
 398                         if (HDR_SIZE - count > 0) {
 399                                 int left = HDR_SIZE - count;
 400
 401                                 memmove(buf, buf + count, left);
 402                                 ptr = buf + left;
 403
 404                                 goto readhdr;
 405                         }
 406
 407                         continue;
 408                 }
 409
 410                 if (hdr->ph_len == 0)
 411                         continue;
 412
 413                 count = hdr->ph_len - HDR_SIZE;
 414 readmore:
 415                 rc = read(fdin, ptr, count);
 416                 if (rc <= 0)
 417                         break;
 418
 419                 ptr += rc;
 420                 count -= rc;
 421                 if (count > 0)
 422                         goto readmore;
 423
 424                 first_bad = 1;
 425
 426                 if ((hdr->ph_subsys && !(subsystem_mask & hdr->ph_subsys)) ||
 427                     (hdr->ph_mask && !(debug_mask & hdr->ph_mask))) {
 428                         dropped++;
 429                         continue;
 430                 }
 431
 432 retry_alloc:
 433                 line = malloc(sizeof(*line));
 434                 if (line == NULL) {
 435                         if (linev) {
 436                                 fprintf(stderr, "error: line malloc(%u): "
 437                                         "printing accumulated records\n",
 438                                         (unsigned int)sizeof(*line));
 439                                 print_rec(&linev, kept, fdout);
 440
 441                                 goto retry_alloc;
 442                         }
 443                         fprintf(stderr, "error: line malloc(%u): exiting\n",
 444                                 (unsigned int)sizeof(*line));
 445                         break;
 446                 }
 447
 448                 line->hdr = malloc(hdr->ph_len + 1);
 449                 if (line->hdr == NULL) {
 450                         free(line);
 451                         if (linev) {
 452                                 fprintf(stderr, "error: hdr malloc(%u): "
 453                                         "printing accumulated records\n",
 454                                         hdr->ph_len + 1);
 455                                 print_rec(&linev, kept, fdout);
 456
 457                                 goto retry_alloc;
 458                         }
 459                         fprintf(stderr, "error: hdr malloc(%u): exiting\n",
 460                                         hdr->ph_len + 1);
 461                         break;
 462                 }
 463
 464                 ptr = (void *)line->hdr;
 465                 memcpy(line->hdr, buf, hdr->ph_len);
 466                 ptr[hdr->ph_len] = '\0';
 467
 468                 ptr += sizeof(*hdr);
 469                 line->file = ptr;
 470                 ptr += strlen(line->file) + 1;
 471                 line->fn = ptr;
 472                 ptr += strlen(line->fn) + 1;
 473                 line->text = ptr;
 474
 475 retry_add:
 476                 if (add_rec(line, &linev, &linev_len, kept) < 0) {
 477                         if (linev) {
 478                                 fprintf(stderr, "error: add_rec[%u] failed; "
 479                                         "print accumulated records\n",
 480                                         linev_len);
 481                                 print_rec(&linev, kept, fdout);
 482
 483                                 goto retry_add;
 484                         }
 485                         fprintf(stderr, "error: add_rec[0] failed; exiting\n");
 486                         break;
 487                 }
 488                 kept++;
 489         }
 490
 491 print:
 492         if (linev)
 493                 print_rec(&linev, kept, fdout);
 494
 495         printf("Debug log: %lu lines, %lu kept, %lu dropped, %lu bad.\n",
 496                 dropped + kept + bad, kept, dropped, bad);
 497
 498         return 0;
 499 }
 500
 501 int jt_dbg_debug_kernel(int argc, char **argv)
 502 {
 503         struct stat     st;
 504         char            filename[PATH_MAX];
 505         int             raw = 0;
 506         int             save_errno;
 507         int             fdin;
 508         int             fdout;
 509         int             rc;
 510
 511         if (argc > 3) {
 512                 fprintf(stderr, "usage: %s [file] [raw]\n", argv[0]);
 513                 return 0;
 514         }
 515
 516         if (argc > 2) {
 517                 raw = atoi(argv[2]);
 518         } else if (argc > 1 && (argv[1][0] == '0' || argv[1][0] == '1')) {
 519                 raw = atoi(argv[1]);
 520                 argc--;
 521         }
 522
 523         /* If we are dumping raw (which means no conversion step to ASCII)
 524          * then dump directly to any supplied filename, otherwise this is
 525          * just a temp file and we dump to the real file at convert time. */
 526         if (argc > 1 && raw) {
 527                 if (strlen(argv[1]) >= sizeof(filename)) {
 528                         fprintf(stderr, "File name too long: %s\n", argv[1]);
 529                         return 1;
 530                 }
 531                 strncpy(filename, argv[1], sizeof(filename));
 532         } else {
 533                 if (snprintf(filename, sizeof(filename), "%s%lu.%u",
 534                              LIBCFS_DEBUG_FILE_PATH_DEFAULT, time(NULL),
 535                              getpid())
 536                     >= sizeof(filename)) {
 537                         fprintf(stderr, "File name too long\n");
 538                         return 1;
 539                 }
 540         }
 541
 542         if (stat(filename, &st) == 0 && S_ISREG(st.st_mode))
 543                 unlink(filename);
 544
 545         fdin = dbg_open_ctlhandle(DUMP_KERNEL_CTL_NAME);
 546         if (fdin < 0) {
 547                 fprintf(stderr, "open(dump_kernel) failed: %s\n",
 548                         strerror(errno));
 549                 return 1;
 550         }
 551
 552         rc = dbg_write_cmd(fdin, filename, strlen(filename));
 553         save_errno = errno;
 554         dbg_close_ctlhandle(fdin);
 555         if (rc != 0) {
 556                 fprintf(stderr, "write(%s) failed: %s\n", filename,
 557                         strerror(save_errno));
 558                 return 1;
 559         }
 560
 561         if (raw)
 562                 return 0;
 563
 564         fdin = open(filename, O_RDONLY);
 565         if (fdin < 0) {
 566                 if (errno == ENOENT) /* no dump file created */
 567                         return 0;
 568                 fprintf(stderr, "fopen(%s) failed: %s\n", filename,
 569                         strerror(errno));
 570                 return 1;
 571         }
 572         if (argc > 1) {
 573                 fdout = open(argv[1], O_WRONLY | O_CREAT | O_TRUNC,
 574                              S_IRUSR | S_IWUSR);
 575                 if (fdout < 0) {
 576                         fprintf(stderr, "fopen(%s) failed: %s\n", argv[1],
 577                                 strerror(errno));
 578                         close(fdin);
 579                         return 1;
 580                 }
 581         } else {
 582                 fdout = fileno(stdout);
 583         }
 584
 585         rc = parse_buffer(fdin, fdout);
 586         close(fdin);
 587         if (argc > 1)
 588                 close(fdout);
 589         if (rc) {
 590                 fprintf(stderr, "parse_buffer failed; leaving tmp file %s "
 591                         "behind.\n", filename);
 592         } else {
 593                 rc = unlink(filename);
 594                 if (rc)
 595                         fprintf(stderr, "dumped successfully, but couldn't "
 596                                 "unlink tmp file %s: %s\n", filename,
 597                                 strerror(errno));
 598         }
 599
 600         return rc;
 601 }
 602
 603 int jt_dbg_debug_file(int argc, char **argv)
 604 {
 605         int fdin;
 606         int fdout;
 607         int rc;
 608
 609         if (argc > 3 || argc < 2) {
 610                 fprintf(stderr, "usage: %s <input> [output]\n", argv[0]);
 611                 return 0;
 612         }
 613
 614         fdin = open(argv[1], O_RDONLY | O_LARGEFILE);
 615         if (fdin < 0) {
 616                 fprintf(stderr, "open(%s) failed: %s\n", argv[1],
 617                         strerror(errno));
 618                 return 1;
 619         }
 620         if (argc > 2) {
 621                 fdout = open(argv[2],
 622                              O_CREAT | O_TRUNC | O_WRONLY | O_LARGEFILE,
 623                              0600);
 624                 if (fdout < 0) {
 625                         fprintf(stderr, "open(%s) failed: %s\n", argv[2],
 626                                 strerror(errno));
 627                         close(fdin);
 628                         return 1;
 629                 }
 630         } else {
 631                 fdout = fileno(stdout);
 632         }
 633
 634         rc = parse_buffer(fdin, fdout);
 635
 636         close(fdin);
 637         if (fdout != fileno(stdout))
 638                 close(fdout);
 639
 640         return rc;
 641 }
 642
 643 const char debug_daemon_usage[] = "usage: %s {start file [MB]|stop}\n";
 644
 645 int jt_dbg_debug_daemon(int argc, char **argv)
 646 {
 647         int rc;
 648         int fd;
 649
 650         if (argc <= 1) {
 651                 fprintf(stderr, debug_daemon_usage, argv[0]);
 652                 return 1;
 653         }
 654
 655         fd = dbg_open_ctlhandle(DAEMON_CTL_NAME);
 656         if (fd < 0)
 657                 return -1;
 658
 659         rc = -1;
 660         if (strcasecmp(argv[1], "start") == 0) {
 661                 if (argc < 3 || argc > 4 ||
 662                     (argc == 4 && strlen(argv[3]) > 5)) {
 663                         fprintf(stderr, debug_daemon_usage, argv[0]);
 664                         goto out;
 665                 }
 666                 if (argc == 4) {
 667                         char             buf[12];
 668                         const long       min_size = 10;
 669                         const long       max_size = 20480;
 670                         long             size;
 671                         char            *end;
 672
 673                         size = strtoul(argv[3], &end, 0);
 674                         if (size < min_size ||
 675                             size > max_size ||
 676                             *end != 0) {
 677                                 fprintf(stderr, "size %s invalid, must be in "
 678                                         "the range %ld-%ld MB\n", argv[3],
 679                                         min_size, max_size);
 680                                 goto out;
 681                         }
 682                         snprintf(buf, sizeof(buf), "size=%ld", size);
 683                         rc = dbg_write_cmd(fd, buf, strlen(buf));
 684
 685                         if (rc != 0) {
 686                                 fprintf(stderr, "set %s failed: %s\n",
 687                                         buf, strerror(errno));
 688                                 goto out;
 689                         }
 690                 }
 691
 692                 rc = dbg_write_cmd(fd, argv[2], strlen(argv[2]));
 693                 if (rc != 0) {
 694                         fprintf(stderr, "start debug_daemon on %s failed: %s\n",
 695                                 argv[2], strerror(errno));
 696                         goto out;
 697                 }
 698                 rc = 0;
 699                 goto out;
 700         }
 701         if (strcasecmp(argv[1], "stop") == 0) {
 702                 rc = dbg_write_cmd(fd, "stop", 4);
 703                 if (rc != 0) {
 704                         fprintf(stderr, "stopping debug_daemon failed: %s\n",
 705                                 strerror(errno));
 706                         goto out;
 707                 }
 708
 709                 rc = 0;
 710                 goto out;
 711         }
 712
 713         fprintf(stderr, debug_daemon_usage, argv[0]);
 714         rc = -1;
 715 out:
 716         dbg_close_ctlhandle(fd);
 717         return rc;
 718 }
 719
 720 int jt_dbg_clear_debug_buf(int argc, char **argv)
 721 {
 722         int                      rc;
 723         struct libcfs_ioctl_data data;
 724
 725         if (argc != 1) {
 726                 fprintf(stderr, "usage: %s\n", argv[0]);
 727                 return 0;
 728         }
 729
 730         memset(&data, 0, sizeof(data));
 731         if (libcfs_ioctl_pack(&data, &buf, max) != 0) {
 732                 fprintf(stderr, "libcfs_ioctl_pack failed.\n");
 733                 return -1;
 734         }
 735
 736         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_CLEAR_DEBUG, buf);
 737         if (rc) {
 738                 fprintf(stderr, "IOC_LIBCFS_CLEAR_DEBUG failed: %s\n",
 739                         strerror(errno));
 740                 return -1;
 741         }
 742         return 0;
 743 }
 744
 745 int jt_dbg_mark_debug_buf(int argc, char **argv)
 746 {
 747         static char              scratch[MAX_MARK_SIZE] = "";
 748         struct libcfs_ioctl_data data;
 749         char                    *text;
 750         int                      rc;
 751
 752         memset(&data, 0, sizeof(data));
 753
 754         if (argc > 1) {
 755                 int count, max_size = sizeof(scratch) - 1;
 756
 757                 strncpy(scratch, argv[1], max_size);
 758                 max_size -= strlen(argv[1]);
 759                 for (count = 2; (count < argc) && (max_size > 1); count++) {
 760                         strncat(scratch, " ", max_size);
 761                         max_size -= 1;
 762                         strncat(scratch, argv[count], max_size);
 763                         max_size -= strlen(argv[count]);
 764                 }
 765                 scratch[sizeof(scratch) - 1] = '\0';
 766                 text = scratch;
 767         } else {
 768                 time_t now = time(NULL);
 769                 text = ctime(&now);
 770         }
 771
 772         data.ioc_inllen1 = strlen(text) + 1;
 773         data.ioc_inlbuf1 = text;
 774
 775         if (libcfs_ioctl_pack(&data, &buf, max) != 0) {
 776                 fprintf(stderr, "libcfs_ioctl_pack failed.\n");
 777                 return -1;
 778         }
 779
 780         rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_MARK_DEBUG, buf);
 781         if (rc) {
 782                 fprintf(stderr, "IOC_LIBCFS_MARK_DEBUG failed: %s\n",
 783                         strerror(errno));
 784                 return -1;
 785         }
 786         return 0;
 787 }
 788
 789 static struct mod_paths {
 790         char *name, *path;
 791 } mod_paths[] = {
 792         { "libcfs", "libcfs/libcfs" },
 793         { "lnet", "lnet/lnet" },
 794         { "ko2iblnd", "lnet/klnds/o2iblnd" },
 795         { "kgnilnd", "lnet/klnds/gnilnd"},
 796         { "ksocklnd", "lnet/klnds/socklnd" },
 797         { "obdclass", "lustre/obdclass" },
 798         { "llog_test", "lustre/obdclass" },
 799         { "ptlrpc_gss", "lustre/ptlrpc/gss" },
 800         { "ptlrpc", "lustre/ptlrpc" },
 801         { "gks", "lustre/sec/gks" },
 802         { "gkc", "lustre/sec/gks" },
 803         { "ost", "lustre/ost" },
 804         { "osc", "lustre/osc" },
 805         { "mds", "lustre/mds" },
 806         { "mdc", "lustre/mdc" },
 807         { "lustre", "lustre/llite" },
 808         { "llite_lloop", "lustre/llite" },
 809         { "ldiskfs", "ldiskfs" },
 810         { "obdecho", "lustre/obdecho" },
 811         { "ldlm", "lustre/ldlm" },
 812         { "obdfilter", "lustre/obdfilter" },
 813         { "lov", "lustre/lov" },
 814         { "lmv", "lustre/lmv" },
 815         { "lquota", "lustre/quota" },
 816         { "mgs", "lustre/mgs" },
 817         { "mgc", "lustre/mgc" },
 818         { "mdt", "lustre/mdt" },
 819         { "mdd", "lustre/mdd" },
 820         { "osd", "lustre/osd" },
 821         { "cmm", "lustre/cmm" },
 822         {"fid", "lustre/fid"},
 823         {"fld", "lustre/fld"},
 824         {"lod", "lustre/lod"},
 825         {"osp", "lustre/osp"},
 826         { "lfsck", "lustre/lfsck" },
 827         {NULL, NULL}
 828 };
 829
 830 int jt_dbg_modules(int argc, char **argv)
 831 {
 832         struct mod_paths *mp;
 833         char             *path = "";
 834         const char       *proc = "/proc/modules";
 835         char              modname[128];
 836         char              buf[4096];
 837         unsigned long     modaddr;
 838         FILE             *file;
 839
 840         if (argc >= 2)
 841                 path = argv[1];
 842         if (argc > 3) {
 843                 printf("%s [path] [kernel]\n", argv[0]);
 844                 return 0;
 845         }
 846
 847         file = fopen(proc, "r");
 848         if (!file) {
 849                 printf("failed open %s: %s\n", proc, strerror(errno));
 850                 return 0;
 851         }
 852
 853         while (fgets(buf, sizeof(buf), file) != NULL) {
 854                 if (sscanf(buf, "%s %*s %*s %*s %*s %lx",
 855                            modname, &modaddr) == 2) {
 856                         for (mp = mod_paths; mp->name != NULL; mp++) {
 857                                 if (!strcmp(mp->name, modname))
 858                                         break;
 859                         }
 860                         if (mp->name) {
 861                                 printf("add-symbol-file %s%s%s/%s.o 0x%0lx\n",
 862                                         path, path[0] ? "/" : "",
 863                                         mp->path, mp->name, modaddr);
 864                         }
 865                 }
 866         }
 867
 868         fclose(file);
 869         return 0;
 870 }