lnet/libcfs/debug.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (C) 2002 Cluster File Systems, Inc.
   5  *   Author: Phil Schwan <phil@clusterfs.com>
   6  *
   7  *   This file is part of Portals, http://www.sf.net/projects/sandiaportals/
   8  *
   9  *   Portals is free software; you can redistribute it and/or
  10  *   modify it under the terms of version 2.1 of the GNU Lesser General
  11  *   Public License as published by the Free Software Foundation.
  12  *
  13  *   Portals is distributed in the hope that it will be useful,
  14  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  *   GNU Lesser General Public License for more details.
  17  *
  18  *   You should have received a copy of the GNU Lesser General Public
  19  *   License along with Portals; if not, write to the Free Software
  20  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21  */
  22
  23 #define EXPORT_SYMTAB
  24
  25 #include <linux/config.h>
  26 #include <linux/module.h>
  27 #include <linux/kmod.h>
  28 #include <linux/notifier.h>
  29 #include <linux/kernel.h>
  30 #include <linux/kernel.h>
  31 #include <linux/mm.h>
  32 #include <linux/string.h>
  33 #include <linux/stat.h>
  34 #include <linux/errno.h>
  35 #include <linux/smp_lock.h>
  36 #include <linux/unistd.h>
  37 #include <linux/interrupt.h>
  38 #include <asm/system.h>
  39 #include <asm/uaccess.h>
  40 #include <linux/completion.h>
  41
  42 #include <linux/fs.h>
  43 #include <linux/stat.h>
  44 #include <asm/uaccess.h>
  45 #include <asm/segment.h>
  46 #include <linux/miscdevice.h>
  47
  48 # define DEBUG_SUBSYSTEM S_PORTALS
  49
  50 #include <linux/kp30.h>
  51
  52 #define DEBUG_OVERFLOW 1024
  53 static char *debug_buf = NULL;
  54 static unsigned long debug_size = 0;
  55 static atomic_t debug_off_a = ATOMIC_INIT(0);
  56 static int debug_wrapped;
  57 wait_queue_head_t debug_ctlwq;
  58 #define DAEMON_SND_SIZE      (64 << 10)
  59
  60 /*
  61  * used by the daemon to keep track the offset into debug_buffer for the next
  62  * write to the file.  Usually, the daemon is to write out buffer
  63  * from debug_daemon_next_write upto debug_off
  64  *  variable usage
  65  *      Reader - portals_debug_msg()
  66  *      Writer - portals_debug_daemon()
  67  *               portals_debug_daemon_start() during daemon init time
  68  *               portals_debug_daemon_continue() to reset to debug_off
  69  *               portals_debug_clear_buffer() reset to debug_off for clear
  70  *      Note that *_start(), *_continue() & *clear_buffer() should serialized;
  71  */
  72 static atomic_t   debug_daemon_next_write;
  73
  74 /*
  75  * A debug_daemon can be in following states
  76  *      stopped - stopped state means there is no debug_daemon running.
  77  *                accordingly, it must be in paused state
  78  *                a daemon is in !stopped && !paused state after
  79  *                "lctl debug_daemon start" creates debug_daemon successfully
  80  *                Variable Usage
  81  *                      Reader - portals_debug_daemon()
  82  *                               portals_debug_set_daemon() routines
  83  *                      Writer - portals_debug_set_daemon() routines
  84  *                              portals_debug_daemon() on IO error
  85  *      paused -  a debug_daemon state is changed from !paused into paused
  86  *                when "lctl debug_daemon paused" is issued
  87  *                "lctl debug_daemon continue" gets a daemon into !paused mode
  88  *                      Reader - portals_debug_set_daemon() routines
  89  *                               portals_debug_msg()
  90  *                      Writer - portals_debug_set_daemon() on init
  91  *                               portals_debug_daemon()
  92  *
  93  *        Daemon  state diagram.
  94  *                      (stopped, paused)
  95  *                              |  <-- debug_daemon start
  96  *                              V
  97  *                      (!stopped, !paused)
  98  *                              |  <-- debug_daemon pause
  99  *                              V
 100  *                      (!stopped, paused)
 101  *                              |  <-- debug_daemon continue
 102  *                              V
 103  *                      (!stopped, !paused)
 104  *                              |  <-- debug_daemon stop
 105  *                              V
 106  *                      (stopped, paused)
 107  *      Overlapped - this is a state when CDEBUG is too fast for the daemon to
 108  *                   write out the debug_bufferr.  That is, debug_off is to
 109  *                   overlap debug_daemon_next_write;
 110  *                     Reader - portals_debug_msg()
 111  *                     Writer - portals_debug_msg()
 112  */
 113
 114 /*
 115  * Description on Trace Daemon Synchronization
 116  *
 117  * Three categories of code are synchronizing between each other
 118  * 1.   lctl, portals_debug_set_daemon(), the user debug control code,
 119  *      as well as portals_debug_clear_buffer()
 120  * 2.   CDEBUG, portals_debug_msg(), the debug put messages routine
 121  * 3.   Daemon, portals_debug_daemon(), to write out debug log file
 122  *
 123  *
 124  * Three different controls for synchronizations
 125  *
 126  * 1.   debug_daemon_semaphore
 127  *      The usage of this semaphore is to serialize multiple lctl controls
 128  *      in manipulating debug daemon state.  The semaphore serves as the
 129  *      gatekeeper to allow only one user control thread, at any giving time,
 130  *      to access debug daemon state and keeps the other user control requests
 131  *      in wait state until the current control request is serviced.
 132  *
 133  * 2.   wait_queue_head_t lctl (paired with lctl_event flag)
 134  *      Lctl event is the event between portals_debug_set_daemon() and
 135  *      portals_debug_daemon().  Lctl is an indicator for portals_debug_daemon()
 136  *      to flush data out to file.  portals_debug_daemon() is to use lctl event
 137  *      as signal channel to wakeup portals_debug_set_daemon() upon flush
 138  *      operation is done.
 139  *
 140  *      Producer :
 141  *              portals_debug_daemon() uses to wake up
 142  *              portals_debug_set_daemon(), pause and stop, routines
 143  *      Consumer :
 144  *              portals_debug_set_daemon(), stop and pause operations,
 145  *              wait and sleep on the event
 146  *
 147  * 3.   wait_queue_head_t daemon (paired with daemon_event flag)
 148  *      This is an event channel to wakeup portals_debug_daemon.  Daemon
 149  *      wakes up to run whenever there is an event posted.   Daemon handles
 150  *      2 types of operations . 1. Writes data out to debug file, 2. Flushes
 151  *      file and terminates base on lctl event.
 152  *      File operation -
 153  *              Daemon is normally in a sleep state.
 154  *              Daemon is woken up through daemon event whenever CDEBUG is
 155  *              putting data over any 64K boundary.
 156  *      File flush and termination -
 157  *              On portals_debug_daemon_stop/pause() operations, lctl control
 158  *              is to wake up daemon through daemon event.
 159  *
 160  *      We can't use sleep_on() and wake_up() to replace daemon event because
 161  *      portals_debug_daemon() must catch the wakeup operation posted by
 162  *      portals_debug_daemon_stop/pause().  Otherwise, stop and pause may
 163  *      stuck in lctl wait event.
 164  *
 165  *      Producer :
 166  *           a. portals_debug_daemon_pause() and portals_debug_daemon_stop()
 167  *              uses the event to wake up portals_debug_daemon()
 168  *           b. portals_debug_msg() uses the event to wake up
 169  *              portals_debug_daemon() whenever the data output is acrossing
 170  *              a 64K bytes boundary.
 171  *      Consumer :
 172  *              portals_debug_daemon() wakes up upon daemon event.
 173  *
 174  * Sequence for portals_debug_daemon_stop() operation
 175  *
 176  * _Portals_debug_daemon_stop()_          _Daemon_
 177  *                                      Wait_event(daemon) or running
 178  *      Paused = 1;
 179  *      Wakeup_event (daemon)
 180  *      Wait_event(lctl)
 181  *                                      Set force_flush flag if lctlevnt
 182  *                                      Flush data
 183  *                                      Wakeup_event (lctl)
 184  *                                      Wait_event(daemon)
 185  *      Stopped = 1;
 186  *      Wakeup_event (daemon)
 187  *      Wait_event(lctl)
 188  *                                      Exit daemon loop if (Stopped)
 189  *                                      Wakeup_event (lctl)
 190  *                                      Exit
 191  *      Return to user application
 192  *
 193  *
 194  * _Portals_debug_msg()_                  _Daemon_
 195  *                                      Wait_event(daemon) or running
 196  *      If (WriteStart<64K<WriteEnd)
 197  *         Wakeup_event(daemon)
 198  *                                      Do file IO
 199  *                                      Wait_event(daemon)
 200  */
 201 struct debug_daemon_state {
 202         unsigned long overlapped;
 203         unsigned long stopped;
 204         atomic_t paused;
 205         unsigned long   lctl_event;     /* event for lctl */
 206         wait_queue_head_t lctl;
 207         unsigned long   daemon_event;   /* event for daemon */
 208         wait_queue_head_t daemon;
 209 };
 210 static struct debug_daemon_state debug_daemon_state;
 211 static DECLARE_MUTEX(debug_daemon_semaphore);
 212
 213 static loff_t daemon_file_size_limit;
 214 char debug_daemon_file_path[1024] = "";
 215
 216 spinlock_t portals_debug_lock = SPIN_LOCK_UNLOCKED;
 217 char debug_file_path[1024] = "/tmp/lustre-log";
 218 char debug_file_name[1024];
 219 int handled_panic; /* to avoid recursive calls to notifiers */
 220 char portals_upcall[1024] = "/usr/lib/lustre/portals_upcall";
 221
 222
 223 int portals_do_debug_dumplog(void *arg)
 224 {
 225         struct file *file;
 226         void *journal_info;
 227         int rc;
 228         mm_segment_t oldfs;
 229         unsigned long debug_off;
 230
 231         kportal_daemonize("");
 232
 233         reparent_to_init();
 234         journal_info = current->journal_info;
 235         current->journal_info = NULL;
 236         sprintf(debug_file_name, "%s.%ld", debug_file_path, CURRENT_TIME);
 237         file = filp_open(debug_file_name, O_CREAT|O_TRUNC|O_RDWR, 0644);
 238
 239         if (!file || IS_ERR(file)) {
 240                 CERROR("cannot open %s for dumping: %ld\n", debug_file_name,
 241                        PTR_ERR(file));
 242                 GOTO(out, PTR_ERR(file));
 243         } else {
 244                 printk(KERN_ALERT "dumping log to %s ... writing ...\n",
 245                        debug_file_name);
 246         }
 247
 248         debug_off = atomic_read(&debug_off_a);
 249         oldfs = get_fs();
 250         set_fs(get_ds());
 251         if (debug_wrapped) {
 252                 rc = file->f_op->write(file, debug_buf + debug_off + 1,
 253                                        debug_size-debug_off-1, &file->f_pos);
 254                 rc += file->f_op->write(file, debug_buf, debug_off + 1,
 255                                         &file->f_pos);
 256         } else {
 257                 rc = file->f_op->write(file, debug_buf, debug_off,&file->f_pos);
 258         }
 259         printk("wrote %d bytes\n", rc);
 260         set_fs(oldfs);
 261
 262         rc = file->f_op->fsync(file, file->f_dentry, 1);
 263         if (rc)
 264                 CERROR("sync returns %d\n", rc);
 265         filp_close(file, 0);
 266 out:
 267         current->journal_info = journal_info;
 268         wake_up(&debug_ctlwq);
 269         return 0;
 270 }
 271
 272 int portals_debug_daemon(void *arg)
 273 {
 274         struct file *file;
 275         void *journal_info;
 276         mm_segment_t oldfs;
 277         unsigned long force_flush = 0;
 278         unsigned long size, off, flags;
 279         int rc;
 280
 281         kportal_daemonize("ldebug_daemon");
 282         reparent_to_init();
 283         journal_info = current->journal_info;
 284         current->journal_info = NULL;
 285
 286         file = filp_open(debug_daemon_file_path,
 287                          O_CREAT|O_TRUNC|O_RDWR|O_LARGEFILE, 0644);
 288
 289         if (!file || IS_ERR(file)) {
 290                 CERROR("cannot open %s for logging", debug_daemon_file_path);
 291                 GOTO(out1, PTR_ERR(file));
 292         } else {
 293                 printk(KERN_ALERT "daemon dumping log to %s ... writing ...\n",
 294                        debug_daemon_file_path);
 295         }
 296
 297         debug_daemon_state.overlapped = 0;
 298         debug_daemon_state.stopped = 0;
 299
 300         spin_lock_irqsave(&portals_debug_lock, flags);
 301         off = atomic_read(&debug_off_a) + 1;
 302         if (debug_wrapped)
 303                 off = (off >= debug_size)? 0 : off;
 304         else
 305                 off = 0;
 306         atomic_set(&debug_daemon_next_write, off);
 307         atomic_set(&debug_daemon_state.paused, 0);
 308         spin_unlock_irqrestore(&portals_debug_lock, flags);
 309
 310         oldfs = get_fs();
 311         set_fs(KERNEL_DS);
 312         while (1) {
 313                 unsigned long ending;
 314                 unsigned long start, tail;
 315                 long delta;
 316
 317                 debug_daemon_state.daemon_event = 0;
 318
 319                 ending = atomic_read(&debug_off_a);
 320                 start = atomic_read(&debug_daemon_next_write);
 321
 322                 /* check if paused is imposed by lctl ? */
 323                 force_flush = !debug_daemon_state.lctl_event;
 324
 325                 delta = ending - start;
 326                 tail = debug_size - start;
 327                 size = (delta >= 0) ? delta : tail;
 328                 while (size && (force_flush || (delta < 0) ||
 329                                 (size >= DAEMON_SND_SIZE))) {
 330                         if (daemon_file_size_limit) {
 331                                int ssize = daemon_file_size_limit - file->f_pos;
 332                                if (size > ssize)
 333                                         size = ssize;
 334                         }
 335
 336                         rc = file->f_op->write(file, debug_buf+start,
 337                                                size, &file->f_pos);
 338                         if (rc < 0) {
 339                                 printk(KERN_ALERT
 340                                            "Debug_daemon write error %d\n", rc);
 341                                 goto out;
 342                         }
 343                         start += rc;
 344                         delta = ending - start;
 345                         tail = debug_size - start;
 346                         if (tail == 0)
 347                                 start = 0;
 348                         if (delta >= 0)
 349                                 size = delta;
 350                         else
 351                                 size = (tail == 0) ? ending : tail;
 352                         if (daemon_file_size_limit == file->f_pos) {
 353                                 // file wrapped around
 354                                 file->f_pos = 0;
 355                         }
 356                 }
 357                 atomic_set(&debug_daemon_next_write, start);
 358                 if (force_flush) {
 359                         rc = file->f_op->fsync(file, file->f_dentry, 1);
 360                         if (rc < 0) {
 361                                 printk(KERN_ALERT
 362                                        "Debug_daemon sync error %d\n", rc);
 363                                 goto out;
 364                         }
 365                         if (debug_daemon_state.stopped)
 366                                break;
 367                         debug_daemon_state.lctl_event = 1;
 368                         wake_up(&debug_daemon_state.lctl);
 369                 }
 370                 wait_event(debug_daemon_state.daemon,
 371                            debug_daemon_state.daemon_event);
 372                 }
 373 out:
 374         atomic_set(&debug_daemon_state.paused, 1);
 375         debug_daemon_state.stopped = 1;
 376         set_fs(oldfs);
 377         filp_close(file, 0);
 378         current->journal_info = journal_info;
 379 out1:
 380         debug_daemon_state.lctl_event = 1;
 381         wake_up(&debug_daemon_state.lctl);
 382         return 0;
 383 }
 384
 385 void portals_debug_print(void)
 386 {
 387         unsigned long dumplen = 64 * 1024;
 388         char *start1, *start2;
 389         char *end1, *end2;
 390         unsigned long debug_off = atomic_read(&debug_off_a);
 391
 392         start1 = debug_buf + debug_off - dumplen;
 393         if (start1 < debug_buf) {
 394                 start1 += debug_size;
 395                 end1 = debug_buf + debug_size - 1;
 396                 start2 = debug_buf;
 397                 end2 = debug_buf + debug_off;
 398         } else {
 399                 end1 = debug_buf + debug_off;
 400                 start2 = debug_buf + debug_off;
 401                 end2 = debug_buf + debug_off;
 402         }
 403
 404         while (start1 < end1) {
 405                 int count = MIN(1024, end1 - start1);
 406                 printk("%*s", count, start1);
 407                 start1 += 1024;
 408         }
 409         while (start2 < end2) {
 410                 int count = MIN(1024, end2 - start2);
 411                 printk("%*s", count, start2);
 412                 start2 += 1024;
 413         }
 414 }
 415
 416 void portals_debug_dumplog(void)
 417 {
 418         int rc;
 419         ENTRY;
 420
 421         init_waitqueue_head(&debug_ctlwq);
 422
 423         rc = kernel_thread(portals_do_debug_dumplog,
 424                            NULL, CLONE_VM | CLONE_FS | CLONE_FILES);
 425         if (rc < 0) {
 426                 printk(KERN_ERR "cannot start dump thread\n");
 427                 return;
 428         }
 429         sleep_on(&debug_ctlwq);
 430 }
 431
 432 int portals_debug_daemon_start(char *file, unsigned int size)
 433 {
 434         int rc;
 435
 436         if (!debug_daemon_state.stopped)
 437                 return -EALREADY;
 438
 439         if (file != NULL)
 440                 strncpy(debug_daemon_file_path, file, 1024);
 441
 442         init_waitqueue_head(&debug_daemon_state.lctl);
 443         init_waitqueue_head(&debug_daemon_state.daemon);
 444
 445         daemon_file_size_limit = size << 20;
 446
 447         debug_daemon_state.lctl_event = 0;
 448         rc = kernel_thread(portals_debug_daemon, NULL, 0);
 449         if (rc < 0) {
 450                 printk(KERN_ERR "cannot start debug daemon thread\n");
 451                 strncpy(debug_daemon_file_path, "\0", 1);
 452                 return rc;
 453         }
 454         wait_event(debug_daemon_state.lctl, debug_daemon_state.lctl_event);
 455         return 0;
 456 }
 457
 458 int portals_debug_daemon_pause(void)
 459 {
 460         if (atomic_read(&debug_daemon_state.paused))
 461                 return -EALREADY;
 462
 463         atomic_set(&debug_daemon_state.paused, 1);
 464         debug_daemon_state.lctl_event = 0;
 465         debug_daemon_state.daemon_event = 1;
 466         wake_up(&debug_daemon_state.daemon);
 467         wait_event(debug_daemon_state.lctl, debug_daemon_state.lctl_event);
 468         return 0;
 469 }
 470
 471 int portals_debug_daemon_continue(void)
 472 {
 473         if (!atomic_read(&debug_daemon_state.paused))
 474                 return -EINVAL;
 475         if (debug_daemon_state.stopped)
 476                 return -EINVAL;
 477
 478         debug_daemon_state.overlapped = 0;
 479         atomic_set(&debug_daemon_next_write, atomic_read(&debug_off_a));
 480         atomic_set(&debug_daemon_state.paused, 0);
 481         return 0;
 482 }
 483
 484 int portals_debug_daemon_stop(void)
 485 {
 486         if (debug_daemon_state.stopped)
 487                 return -EALREADY;
 488
 489         if (!atomic_read(&debug_daemon_state.paused))
 490                 portals_debug_daemon_pause();
 491
 492         debug_daemon_state.lctl_event = 0;
 493         debug_daemon_state.stopped = 1;
 494
 495         debug_daemon_state.daemon_event = 1;
 496         wake_up(&debug_daemon_state.daemon);
 497         wait_event(debug_daemon_state.lctl, debug_daemon_state.lctl_event);
 498
 499         debug_daemon_file_path[0] = '\0';
 500         return 0;
 501 }
 502
 503 int portals_debug_set_daemon(unsigned int cmd, unsigned int length,
 504                              char *filename, unsigned int size)
 505 {
 506         int rc = -EINVAL;
 507
 508         down(&debug_daemon_semaphore);
 509         switch (cmd) {
 510                 case DEBUG_DAEMON_START:
 511                         if (length && (filename[length -1] != '\0')) {
 512                                 CERROR("Invalid filename for debug_daemon\n");
 513                                 rc = -EINVAL;
 514                                 break;
 515                         }
 516                         rc = portals_debug_daemon_start(filename, size);
 517                         break;
 518                 case DEBUG_DAEMON_STOP:
 519                         rc = portals_debug_daemon_stop();
 520                         break;
 521                 case DEBUG_DAEMON_PAUSE:
 522                         rc = portals_debug_daemon_pause();
 523                         break;
 524                 case DEBUG_DAEMON_CONTINUE:
 525                         rc = portals_debug_daemon_continue();
 526                         break;
 527                 default:
 528                         CERROR("unknown set_daemon cmd\n");
 529         }
 530         up(&debug_daemon_semaphore);
 531         return rc;
 532 }
 533
 534 static int panic_dumplog(struct notifier_block *self, unsigned long unused1,
 535                          void *unused2)
 536 {
 537         if (handled_panic)
 538                 return 0;
 539         else
 540                 handled_panic = 1;
 541
 542         if (in_interrupt()) {
 543                 portals_debug_print();
 544                 return 0;
 545         }
 546
 547         while (current->lock_depth >= 0)
 548                 unlock_kernel();
 549         portals_debug_dumplog();
 550         return 0;
 551 }
 552
 553 static struct notifier_block lustre_panic_notifier = {
 554         notifier_call :     panic_dumplog,
 555         next :              NULL,
 556         priority :          10000
 557 };
 558
 559 int portals_debug_init(unsigned long bufsize)
 560 {
 561         unsigned long debug_off = atomic_read(&debug_off_a);
 562         if (debug_buf != NULL)
 563                 return -EALREADY;
 564
 565         atomic_set(&debug_daemon_state.paused, 1);
 566         debug_daemon_state.stopped = 1;
 567
 568         debug_buf = vmalloc(bufsize + DEBUG_OVERFLOW);
 569         if (debug_buf == NULL)
 570                 return -ENOMEM;
 571         memset(debug_buf, 0, debug_size);
 572         debug_wrapped = 0;
 573
 574         //printk(KERN_INFO "Portals: allocated %lu byte debug buffer at %p.\n",
 575                //bufsize, debug_buf);
 576         atomic_set(&debug_off_a, debug_off);
 577         notifier_chain_register(&panic_notifier_list, &lustre_panic_notifier);
 578         debug_size = bufsize;
 579
 580         return 0;
 581 }
 582
 583 int portals_debug_cleanup(void)
 584 {
 585         notifier_chain_unregister(&panic_notifier_list, &lustre_panic_notifier);
 586         if (debug_buf == NULL)
 587                 return -EINVAL;
 588
 589         down(&debug_daemon_semaphore);
 590         portals_debug_daemon_stop();
 591
 592         vfree(debug_buf);
 593         atomic_set(&debug_off_a, 0);
 594         up(&debug_daemon_semaphore);
 595
 596         return 0;
 597 }
 598
 599 int portals_debug_clear_buffer(void)
 600 {
 601         unsigned long flags;
 602         unsigned long state;
 603
 604         if (debug_buf == NULL)
 605                 return -EINVAL;
 606
 607         down(&debug_daemon_semaphore);
 608         state = atomic_read(&debug_daemon_state.paused);
 609         if (!state)
 610                 portals_debug_daemon_pause();
 611         spin_lock_irqsave(&portals_debug_lock, flags);
 612         atomic_set(&debug_off_a, 0);
 613         debug_wrapped = 0;
 614         atomic_set(&debug_daemon_next_write, 0);
 615         debug_daemon_state.overlapped = 0;
 616         spin_unlock_irqrestore(&portals_debug_lock, flags);
 617
 618         if (!state)
 619                 atomic_set(&debug_daemon_state.paused, 0);
 620         up(&debug_daemon_semaphore);
 621
 622         return 0;
 623 }
 624
 625 /* Debug markers, although printed by S_PORTALS
 626  * should not be be marked as such.
 627  */
 628 #undef DEBUG_SUBSYSTEM
 629 #define DEBUG_SUBSYSTEM S_UNDEFINED
 630 int portals_debug_mark_buffer(char *text)
 631 {
 632         if (debug_buf == NULL)
 633                 return -EINVAL;
 634
 635         CDEBUG(0, "********************************************************\n");
 636         CDEBUG(0, "DEBUG MARKER: %s\n", text);
 637         CDEBUG(0, "********************************************************\n");
 638
 639         return 0;
 640 }
 641 #undef DEBUG_SUBSYSTEM
 642 #define DEBUG_SUBSYSTEM S_PORTALS
 643
 644 __s32 portals_debug_copy_to_user(char *buf, unsigned long len)
 645 {
 646         int rc;
 647         unsigned long debug_off;
 648         unsigned long flags;
 649
 650         if (len < debug_size)
 651                 return -ENOSPC;
 652
 653         debug_off = atomic_read(&debug_off_a);
 654         spin_lock_irqsave(&portals_debug_lock, flags);
 655         if (debug_wrapped) {
 656                 /* All of this juggling with the 1s is to keep the trailing nul
 657                  * (which falls at debug_buf + debug_off) at the end of what we
 658                  * copy into user space */
 659                 copy_to_user(buf, debug_buf + debug_off + 1,
 660                              debug_size - debug_off - 1);
 661                 copy_to_user(buf + debug_size - debug_off - 1,
 662                              debug_buf, debug_off + 1);
 663                 rc = debug_size;
 664         } else {
 665                 copy_to_user(buf, debug_buf, debug_off);
 666                 rc = debug_off;
 667         }
 668         spin_unlock_irqrestore(&portals_debug_lock, flags);
 669
 670         return rc;
 671 }
 672
 673 /* FIXME: I'm not very smart; someone smarter should make this better. */
 674 void
 675 portals_debug_msg(int subsys, int mask, char *file, const char *fn,
 676                   const int line, unsigned long stack, const char *format, ...)
 677 {
 678         va_list       ap;
 679         unsigned long flags;
 680         int           max_nob;
 681         int           prefix_nob;
 682         int           msg_nob;
 683         struct timeval tv;
 684         unsigned long base_offset;
 685         unsigned long debug_off;
 686
 687         if (debug_buf == NULL) {
 688                 printk("portals_debug_msg: debug_buf is NULL!\n");
 689                 return;
 690         }
 691
 692         spin_lock_irqsave(&portals_debug_lock, flags);
 693         debug_off = atomic_read(&debug_off_a);
 694         if (!atomic_read(&debug_daemon_state.paused)) {
 695                 unsigned long available;
 696                 long delta;
 697                 long v = atomic_read(&debug_daemon_next_write);
 698
 699                 delta = debug_off - v;
 700                 available = (delta>=0) ? debug_size-delta : -delta;
 701                 // Check if we still have enough debug buffer for CDEBUG
 702                 if (available < DAEMON_SND_SIZE) {
 703                         /* Drop CDEBUG packets until enough debug_buffer is
 704                          * available */
 705                         if (debug_daemon_state.overlapped)
 706                                  goto out;
 707                         /* If this is the first time, leave a marker in the
 708                          * output */
 709                         debug_daemon_state.overlapped = 1;
 710                         ap = NULL;
 711                         format = "DEBUG MARKER: Debug buffer overlapped\n";
 712                 } else  /* More space just became available */
 713                         debug_daemon_state.overlapped = 0;
 714         }
 715
 716         max_nob = debug_size - debug_off + DEBUG_OVERFLOW;
 717         if (max_nob <= 0) {
 718                 spin_unlock_irqrestore(&portals_debug_lock, flags);
 719                 printk("logic error in portals_debug_msg: <0 bytes to write\n");
 720                 return;
 721         }
 722
 723         /* NB since we pass a non-zero sized buffer (at least) on the first
 724          * print, we can be assured that by the end of all the snprinting,
 725          * we _do_ have a terminated buffer, even if our message got truncated.
 726          */
 727
 728         do_gettimeofday(&tv);
 729
 730         prefix_nob = snprintf(debug_buf + debug_off, max_nob,
 731                               "%06x:%06x:%d:%lu.%06lu ",
 732                               subsys, mask, smp_processor_id(),
 733                               tv.tv_sec, tv.tv_usec);
 734         max_nob -= prefix_nob;
 735
 736 #if defined(__arch_um__) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,4,20))
 737         msg_nob = snprintf(debug_buf + debug_off + prefix_nob, max_nob,
 738                            "(%s:%d:%s() %d | %d+%lu): ",
 739                            file, line, fn, current->pid,
 740                            current->thread.extern_pid, stack);
 741 #elif defined(__arch_um__) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
 742         msg_nob = snprintf(debug_buf + debug_off + prefix_nob, max_nob,
 743                            "(%s:%d:%s() %d | %d+%lu): ",
 744                            file, line, fn, current->pid,
 745                            current->thread.mode.tt.extern_pid, stack);
 746 #else
 747         msg_nob = snprintf(debug_buf + debug_off + prefix_nob, max_nob,
 748                            "(%s:%d:%s() %d+%lu): ",
 749                            file, line, fn, current->pid, stack);
 750 #endif
 751         max_nob -= msg_nob;
 752
 753         va_start(ap, format);
 754         msg_nob += vsnprintf(debug_buf + debug_off + prefix_nob + msg_nob,
 755                              max_nob, format, ap);
 756         max_nob -= msg_nob;
 757         va_end(ap);
 758
 759         /* Print to console, while msg is contiguous in debug_buf */
 760         /* NB safely terminated see above */
 761         if ((mask & D_EMERG) != 0)
 762                 printk(KERN_EMERG "%s", debug_buf + debug_off + prefix_nob);
 763         if ((mask & D_ERROR) != 0)
 764                 printk(KERN_ERR   "%s", debug_buf + debug_off + prefix_nob);
 765         else if (portal_printk)
 766                 printk("<%d>%s", portal_printk, debug_buf+debug_off+prefix_nob);
 767         base_offset = debug_off & 0xFFFF;
 768
 769         debug_off += prefix_nob + msg_nob;
 770         if (debug_off > debug_size) {
 771                 memcpy(debug_buf, debug_buf + debug_size,
 772                        debug_off - debug_size + 1);
 773                 debug_off -= debug_size;
 774                 debug_wrapped = 1;
 775         }
 776
 777         atomic_set(&debug_off_a, debug_off);
 778         if (!atomic_read(&debug_daemon_state.paused) &&
 779             ((base_offset+prefix_nob+msg_nob) >= DAEMON_SND_SIZE)) {
 780                 debug_daemon_state.daemon_event = 1;
 781                 wake_up(&debug_daemon_state.daemon);
 782         }
 783 out:
 784         spin_unlock_irqrestore(&portals_debug_lock, flags);
 785 }
 786
 787 void portals_debug_set_level(unsigned int debug_level)
 788 {
 789         printk("Setting portals debug level to %08x\n", debug_level);
 790         portal_debug = debug_level;
 791 }
 792
 793 void portals_run_lbug_upcall(char *file, const char *fn, const int line)
 794 {
 795         char *argv[6];
 796         char *envp[3];
 797         char buf[32];
 798         int rc;
 799
 800         ENTRY;
 801         snprintf (buf, sizeof buf, "%d", line);
 802
 803         argv[0] = portals_upcall;
 804         argv[1] = "LBUG";
 805         argv[2] = file;
 806         argv[3] = (char *)fn;
 807         argv[4] = buf;
 808         argv[5] = NULL;
 809
 810         envp[0] = "HOME=/";
 811         envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
 812         envp[2] = NULL;
 813
 814         rc = call_usermodehelper(argv[0], argv, envp);
 815         if (rc < 0) {
 816                 CERROR("Error invoking lbug upcall %s %s %s %s %s: %d; check "
 817                        "/proc/sys/portals/upcall\n",
 818                        argv[0], argv[1], argv[2], argv[3], argv[4], rc);
 819
 820         } else {
 821                 CERROR("Invoked upcall %s %s %s %s %s\n",
 822                        argv[0], argv[1], argv[2], argv[3], argv[4]);
 823         }
 824 }
 825
 826
 827 EXPORT_SYMBOL(portals_debug_dumplog);
 828 EXPORT_SYMBOL(portals_debug_msg);
 829 EXPORT_SYMBOL(portals_debug_set_level);
 830 EXPORT_SYMBOL(portals_run_lbug_upcall);