libcfs/libcfs/tracefile.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * libcfs/libcfs/tracefile.c
  33  *
  34  * Author: Zach Brown <zab@clusterfs.com>
  35  * Author: Phil Schwan <phil@clusterfs.com>
  36  */
  37
  38
  39 #define DEBUG_SUBSYSTEM S_LNET
  40 #define LUSTRE_TRACEFILE_PRIVATE
  41 #include "tracefile.h"
  42
  43 #include <linux/ctype.h>
  44 #include <linux/fs.h>
  45 #include <linux/kthread.h>
  46 #include <linux/pagemap.h>
  47 #include <linux/poll.h>
  48 #include <linux/tty.h>
  49 #include <linux/uaccess.h>
  50 #include <libcfs/linux/linux-fs.h>
  51 #include <libcfs/libcfs.h>
  52
  53 #define TCD_MAX_TYPES                   8
  54
  55 union cfs_trace_data_union (*cfs_trace_data[TCD_MAX_TYPES])[NR_CPUS] __cacheline_aligned;
  56
  57 char *cfs_trace_console_buffers[NR_CPUS][CFS_TCD_TYPE_MAX];
  58 char cfs_tracefile[TRACEFILE_NAME_SIZE];
  59 long long cfs_tracefile_size = CFS_TRACEFILE_SIZE;
  60 static struct tracefiled_ctl trace_tctl;
  61 static DEFINE_MUTEX(cfs_trace_thread_mutex);
  62 static int thread_running = 0;
  63
  64 static atomic_t cfs_tage_allocated = ATOMIC_INIT(0);
  65 static DECLARE_RWSEM(cfs_tracefile_sem);
  66
  67 static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
  68                                         struct cfs_trace_cpu_data *tcd);
  69
  70 /* trace file lock routines */
  71 /* The walking argument indicates the locking comes from all tcd types
  72  * iterator and we must lock it and dissable local irqs to avoid deadlocks
  73  * with other interrupt locks that might be happening. See LU-1311
  74  * for details.
  75  */
  76 int cfs_trace_lock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
  77         __acquires(&tcd->tcd_lock)
  78 {
  79         __LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
  80         if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
  81                 spin_lock_irqsave(&tcd->tcd_lock, tcd->tcd_lock_flags);
  82         else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
  83                 spin_lock_bh(&tcd->tcd_lock);
  84         else if (unlikely(walking))
  85                 spin_lock_irq(&tcd->tcd_lock);
  86         else
  87                 spin_lock(&tcd->tcd_lock);
  88         return 1;
  89 }
  90
  91 void cfs_trace_unlock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
  92         __releases(&tcd->tcd_lock)
  93 {
  94         __LASSERT(tcd->tcd_type < CFS_TCD_TYPE_MAX);
  95         if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
  96                 spin_unlock_irqrestore(&tcd->tcd_lock, tcd->tcd_lock_flags);
  97         else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
  98                 spin_unlock_bh(&tcd->tcd_lock);
  99         else if (unlikely(walking))
 100                 spin_unlock_irq(&tcd->tcd_lock);
 101         else
 102                 spin_unlock(&tcd->tcd_lock);
 103 }
 104
 105 #define cfs_tcd_for_each(tcd, i, j)                                     \
 106         for (i = 0; cfs_trace_data[i]; i++)                             \
 107                 for (j = 0, ((tcd) = &(*cfs_trace_data[i])[j].tcd);     \
 108                      j < num_possible_cpus();                           \
 109                      j++, (tcd) = &(*cfs_trace_data[i])[j].tcd)
 110
 111 #define cfs_tcd_for_each_type_lock(tcd, i, cpu)                         \
 112         for (i = 0; cfs_trace_data[i] &&                                \
 113              (tcd = &(*cfs_trace_data[i])[cpu].tcd) &&                  \
 114              cfs_trace_lock_tcd(tcd, 1); cfs_trace_unlock_tcd(tcd, 1), i++)
 115
 116 enum cfs_trace_buf_type cfs_trace_buf_idx_get(void)
 117 {
 118         if (in_irq())
 119                 return CFS_TCD_TYPE_IRQ;
 120         if (in_softirq())
 121                 return CFS_TCD_TYPE_SOFTIRQ;
 122         return CFS_TCD_TYPE_PROC;
 123 }
 124
 125 static inline struct cfs_trace_cpu_data *
 126 cfs_trace_get_tcd(void)
 127 {
 128         struct cfs_trace_cpu_data *tcd =
 129                 &(*cfs_trace_data[cfs_trace_buf_idx_get()])[get_cpu()].tcd;
 130
 131         cfs_trace_lock_tcd(tcd, 0);
 132
 133         return tcd;
 134 }
 135
 136 static inline void cfs_trace_put_tcd(struct cfs_trace_cpu_data *tcd)
 137 {
 138         cfs_trace_unlock_tcd(tcd, 0);
 139
 140         put_cpu();
 141 }
 142
 143 /* percents to share the total debug memory for each type */
 144 static unsigned int pages_factor[CFS_TCD_TYPE_MAX] = {
 145         80,     /* 80% pages for CFS_TCD_TYPE_PROC */
 146         10,     /* 10% pages for CFS_TCD_TYPE_SOFTIRQ */
 147         10      /* 10% pages for CFS_TCD_TYPE_IRQ */
 148 };
 149
 150 int cfs_tracefile_init_arch(void)
 151 {
 152         struct cfs_trace_cpu_data *tcd;
 153         int i;
 154         int j;
 155
 156         /* initialize trace_data */
 157         memset(cfs_trace_data, 0, sizeof(cfs_trace_data));
 158         for (i = 0; i < CFS_TCD_TYPE_MAX; i++) {
 159                 cfs_trace_data[i] =
 160                         kmalloc_array(num_possible_cpus(),
 161                                       sizeof(union cfs_trace_data_union),
 162                                       GFP_KERNEL);
 163                 if (!cfs_trace_data[i])
 164                         goto out_trace_data;
 165         }
 166
 167         /* arch related info initialized */
 168         cfs_tcd_for_each(tcd, i, j) {
 169                 spin_lock_init(&tcd->tcd_lock);
 170                 tcd->tcd_pages_factor = pages_factor[i];
 171                 tcd->tcd_type = i;
 172                 tcd->tcd_cpu = j;
 173         }
 174
 175         for (i = 0; i < num_possible_cpus(); i++)
 176                 for (j = 0; j < 3; j++) {
 177                         cfs_trace_console_buffers[i][j] =
 178                                 kmalloc(CFS_TRACE_CONSOLE_BUFFER_SIZE,
 179                                         GFP_KERNEL);
 180
 181                         if (!cfs_trace_console_buffers[i][j])
 182                                 goto out_buffers;
 183                 }
 184
 185         return 0;
 186
 187 out_buffers:
 188         for (i = 0; i < num_possible_cpus(); i++)
 189                 for (j = 0; j < 3; j++) {
 190                         kfree(cfs_trace_console_buffers[i][j]);
 191                         cfs_trace_console_buffers[i][j] = NULL;
 192                 }
 193 out_trace_data:
 194         for (i = 0; cfs_trace_data[i]; i++) {
 195                 kfree(cfs_trace_data[i]);
 196                 cfs_trace_data[i] = NULL;
 197         }
 198         pr_err("lnet: Not enough memory\n");
 199         return -ENOMEM;
 200 }
 201
 202 void cfs_tracefile_fini_arch(void)
 203 {
 204         int i;
 205         int j;
 206
 207         for (i = 0; i < num_possible_cpus(); i++)
 208                 for (j = 0; j < 3; j++) {
 209                         kfree(cfs_trace_console_buffers[i][j]);
 210                         cfs_trace_console_buffers[i][j] = NULL;
 211                 }
 212
 213         for (i = 0; cfs_trace_data[i]; i++) {
 214                 kfree(cfs_trace_data[i]);
 215                 cfs_trace_data[i] = NULL;
 216         }
 217 }
 218
 219 static inline struct cfs_trace_page *
 220 cfs_tage_from_list(struct list_head *list)
 221 {
 222         return list_entry(list, struct cfs_trace_page, linkage);
 223 }
 224
 225 static struct cfs_trace_page *cfs_tage_alloc(gfp_t gfp)
 226 {
 227         struct page            *page;
 228         struct cfs_trace_page *tage;
 229
 230         /* My caller is trying to free memory */
 231         if (!in_interrupt() && (current->flags & PF_MEMALLOC))
 232                 return NULL;
 233
 234         /*
 235          * Don't spam console with allocation failures: they will be reported
 236          * by upper layer anyway.
 237          */
 238         gfp |= __GFP_NOWARN;
 239         page = alloc_page(gfp);
 240         if (page == NULL)
 241                 return NULL;
 242
 243         tage = kmalloc(sizeof(*tage), gfp);
 244         if (tage == NULL) {
 245                 __free_page(page);
 246                 return NULL;
 247         }
 248
 249         tage->page = page;
 250         atomic_inc(&cfs_tage_allocated);
 251         return tage;
 252 }
 253
 254 static void cfs_tage_free(struct cfs_trace_page *tage)
 255 {
 256         __LASSERT(tage != NULL);
 257         __LASSERT(tage->page != NULL);
 258
 259         __free_page(tage->page);
 260         kfree(tage);
 261         atomic_dec(&cfs_tage_allocated);
 262 }
 263
 264 static void cfs_tage_to_tail(struct cfs_trace_page *tage,
 265                              struct list_head *queue)
 266 {
 267         __LASSERT(tage != NULL);
 268         __LASSERT(queue != NULL);
 269
 270         list_move_tail(&tage->linkage, queue);
 271 }
 272
 273 /* return a page that has 'len' bytes left at the end */
 274 static struct cfs_trace_page *
 275 cfs_trace_get_tage_try(struct cfs_trace_cpu_data *tcd, unsigned long len)
 276 {
 277         struct cfs_trace_page *tage;
 278
 279         if (tcd->tcd_cur_pages > 0) {
 280                 __LASSERT(!list_empty(&tcd->tcd_pages));
 281                 tage = cfs_tage_from_list(tcd->tcd_pages.prev);
 282                 if (tage->used + len <= PAGE_SIZE)
 283                         return tage;
 284         }
 285
 286         if (tcd->tcd_cur_pages < tcd->tcd_max_pages) {
 287                 if (tcd->tcd_cur_stock_pages > 0) {
 288                         tage = cfs_tage_from_list(tcd->tcd_stock_pages.prev);
 289                         --tcd->tcd_cur_stock_pages;
 290                         list_del_init(&tage->linkage);
 291                 } else {
 292                         tage = cfs_tage_alloc(GFP_ATOMIC);
 293                         if (unlikely(tage == NULL)) {
 294                                 if ((!(current->flags & PF_MEMALLOC) ||
 295                                      in_interrupt()) && printk_ratelimit())
 296                                         pr_warn("Lustre: cannot allocate a tage (%ld)\n",
 297                                                 tcd->tcd_cur_pages);
 298                                 return NULL;
 299                         }
 300                 }
 301
 302                 tage->used = 0;
 303                 tage->cpu = smp_processor_id();
 304                 tage->type = tcd->tcd_type;
 305                 list_add_tail(&tage->linkage, &tcd->tcd_pages);
 306                 tcd->tcd_cur_pages++;
 307
 308                 if (tcd->tcd_cur_pages > 8 && thread_running) {
 309                         struct tracefiled_ctl *tctl = &trace_tctl;
 310                         /*
 311                          * wake up tracefiled to process some pages.
 312                          */
 313                         wake_up(&tctl->tctl_waitq);
 314                 }
 315                 return tage;
 316         }
 317         return NULL;
 318 }
 319
 320 static void cfs_tcd_shrink(struct cfs_trace_cpu_data *tcd)
 321 {
 322         int pgcount = tcd->tcd_cur_pages / 10;
 323         struct page_collection pc;
 324         struct cfs_trace_page *tage;
 325         struct cfs_trace_page *tmp;
 326
 327         /*
 328          * XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
 329          * from here: this will lead to infinite recursion.
 330          */
 331
 332         if (printk_ratelimit())
 333                 pr_warn("Lustre: debug daemon buffer overflowed; discarding 10%% of pages (%d of %ld)\n",
 334                         pgcount + 1, tcd->tcd_cur_pages);
 335
 336         INIT_LIST_HEAD(&pc.pc_pages);
 337
 338         list_for_each_entry_safe(tage, tmp, &tcd->tcd_pages, linkage) {
 339                 if (pgcount-- == 0)
 340                         break;
 341
 342                 list_move_tail(&tage->linkage, &pc.pc_pages);
 343                 tcd->tcd_cur_pages--;
 344         }
 345         put_pages_on_tcd_daemon_list(&pc, tcd);
 346 }
 347
 348 /* return a page that has 'len' bytes left at the end */
 349 static struct cfs_trace_page *cfs_trace_get_tage(struct cfs_trace_cpu_data *tcd,
 350                                                  unsigned long len)
 351 {
 352         struct cfs_trace_page *tage;
 353
 354         /*
 355          * XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
 356          * from here: this will lead to infinite recursion.
 357          */
 358
 359         if (len > PAGE_SIZE) {
 360                 pr_err("LustreError: cowardly refusing to write %lu bytes in a page\n",
 361                        len);
 362                 return NULL;
 363         }
 364
 365         tage = cfs_trace_get_tage_try(tcd, len);
 366         if (tage != NULL)
 367                 return tage;
 368         if (thread_running)
 369                 cfs_tcd_shrink(tcd);
 370         if (tcd->tcd_cur_pages > 0) {
 371                 tage = cfs_tage_from_list(tcd->tcd_pages.next);
 372                 tage->used = 0;
 373                 cfs_tage_to_tail(tage, &tcd->tcd_pages);
 374         }
 375         return tage;
 376 }
 377
 378 static void cfs_set_ptldebug_header(struct ptldebug_header *header,
 379                                     struct libcfs_debug_msg_data *msgdata,
 380                                     unsigned long stack)
 381 {
 382         struct timespec64 ts;
 383
 384         ktime_get_real_ts64(&ts);
 385
 386         header->ph_subsys = msgdata->msg_subsys;
 387         header->ph_mask = msgdata->msg_mask;
 388         header->ph_cpu_id = smp_processor_id();
 389         header->ph_type = cfs_trace_buf_idx_get();
 390         /* y2038 safe since all user space treats this as unsigned, but
 391          * will overflow in 2106
 392          */
 393         header->ph_sec = (u32)ts.tv_sec;
 394         header->ph_usec = ts.tv_nsec / NSEC_PER_USEC;
 395         header->ph_stack = stack;
 396         header->ph_pid = current->pid;
 397         header->ph_line_num = msgdata->msg_line;
 398         header->ph_extern_pid = 0;
 399 }
 400
 401 /**
 402  * tty_write_msg - write a message to a certain tty, not just the console.
 403  * @tty: the destination tty_struct
 404  * @msg: the message to write
 405  *
 406  * tty_write_message is not exported, so write a same function for it
 407  *
 408  */
 409 static void tty_write_msg(struct tty_struct *tty, const char *msg)
 410 {
 411         mutex_lock(&tty->atomic_write_lock);
 412         tty_lock(tty);
 413         if (tty->ops->write && tty->count > 0)
 414                 tty->ops->write(tty, msg, strlen(msg));
 415         tty_unlock(tty);
 416         mutex_unlock(&tty->atomic_write_lock);
 417         wake_up_interruptible_poll(&tty->write_wait, POLLOUT);
 418 }
 419
 420 static void cfs_tty_write_message(const char *prefix, int mask, const char *msg)
 421 {
 422         struct tty_struct *tty;
 423
 424         tty = get_current_tty();
 425         if (!tty)
 426                 return;
 427
 428         tty_write_msg(tty, prefix);
 429         if ((mask & D_EMERG) || (mask & D_ERROR))
 430                 tty_write_msg(tty, "Error");
 431         tty_write_msg(tty, ": ");
 432         tty_write_msg(tty, msg);
 433         tty_kref_put(tty);
 434 }
 435
 436 static void cfs_print_to_console(struct ptldebug_header *hdr, int mask,
 437                                  const char *buf, int len, const char *file,
 438                                  const char *fn)
 439 {
 440         char *prefix = "Lustre";
 441
 442         if (hdr->ph_subsys == S_LND || hdr->ph_subsys == S_LNET)
 443                 prefix = "LNet";
 444
 445         if (mask & D_CONSOLE) {
 446                 if (mask & D_EMERG)
 447                         pr_emerg("%sError: %.*s", prefix, len, buf);
 448                 else if (mask & D_ERROR)
 449                         pr_err("%sError: %.*s", prefix, len, buf);
 450                 else if (mask & D_WARNING)
 451                         pr_warn("%s: %.*s", prefix, len, buf);
 452                 else if (mask & libcfs_printk)
 453                         pr_info("%s: %.*s", prefix, len, buf);
 454         } else {
 455                 if (mask & D_EMERG)
 456                         pr_emerg("%sError: %d:%d:(%s:%d:%s()) %.*s", prefix,
 457                                  hdr->ph_pid, hdr->ph_extern_pid, file,
 458                                  hdr->ph_line_num, fn, len, buf);
 459                 else if (mask & D_ERROR)
 460                         pr_err("%sError: %d:%d:(%s:%d:%s()) %.*s", prefix,
 461                                hdr->ph_pid, hdr->ph_extern_pid, file,
 462                                hdr->ph_line_num, fn, len, buf);
 463                 else if (mask & D_WARNING)
 464                         pr_warn("%s: %d:%d:(%s:%d:%s()) %.*s", prefix,
 465                                 hdr->ph_pid, hdr->ph_extern_pid, file,
 466                                 hdr->ph_line_num, fn, len, buf);
 467                 else if (mask & (D_CONSOLE | libcfs_printk))
 468                         pr_info("%s: %.*s", prefix, len, buf);
 469         }
 470
 471         if (mask & D_TTY)
 472                 cfs_tty_write_message(prefix, mask, buf);
 473 }
 474
 475 int libcfs_debug_msg(struct libcfs_debug_msg_data *msgdata,
 476                      const char *format, ...)
 477 {
 478         struct cfs_trace_cpu_data *tcd = NULL;
 479         struct ptldebug_header header = {0};
 480         struct cfs_trace_page *tage;
 481         /* string_buf is used only if tcd != NULL, and is always set then */
 482         char *string_buf = NULL;
 483         char *debug_buf;
 484         int known_size;
 485         int needed = 85; /* seeded with average message length */
 486         int max_nob;
 487         va_list ap;
 488         int retry;
 489         int mask = msgdata->msg_mask;
 490         char *file = (char *)msgdata->msg_file;
 491         struct cfs_debug_limit_state *cdls = msgdata->msg_cdls;
 492
 493         if (strchr(file, '/'))
 494                 file = strrchr(file, '/') + 1;
 495
 496         tcd = cfs_trace_get_tcd();
 497
 498         /* cfs_trace_get_tcd() grabs a lock, which disables preemption and
 499          * pins us to a particular CPU.  This avoids an smp_processor_id()
 500          * warning on Linux when debugging is enabled.
 501          */
 502         cfs_set_ptldebug_header(&header, msgdata, CDEBUG_STACK());
 503
 504         if (!tcd)                /* arch may not log in IRQ context */
 505                 goto console;
 506
 507         if (tcd->tcd_cur_pages == 0)
 508                 header.ph_flags |= PH_FLAG_FIRST_RECORD;
 509
 510         if (tcd->tcd_shutting_down) {
 511                 cfs_trace_put_tcd(tcd);
 512                 tcd = NULL;
 513                 goto console;
 514         }
 515
 516         known_size = strlen(file) + 1;
 517         if (msgdata->msg_fn)
 518                 known_size += strlen(msgdata->msg_fn) + 1;
 519
 520         if (libcfs_debug_binary)
 521                 known_size += sizeof(header);
 522
 523         /*
 524          * May perform an additional pass to update 'needed' and increase
 525          * tage buffer size to match vsnprintf reported size required
 526          * On the second pass (retry=1) use vscnprintf [which returns
 527          * number of bytes written not including the terminating nul]
 528          * to clarify `needed` is used as number of bytes written
 529          * for the remainder of this function
 530          */
 531         for (retry = 0; retry < 2; retry++) {
 532                 tage = cfs_trace_get_tage(tcd, needed + known_size + 1);
 533                 if (!tage) {
 534                         if (needed + known_size > PAGE_SIZE)
 535                                 mask |= D_ERROR;
 536
 537                         cfs_trace_put_tcd(tcd);
 538                         tcd = NULL;
 539                         goto console;
 540                 }
 541
 542                 string_buf = (char *)page_address(tage->page) +
 543                              tage->used + known_size;
 544
 545                 max_nob = PAGE_SIZE - tage->used - known_size;
 546                 if (max_nob <= 0) {
 547                         pr_emerg("LustreError: negative max_nob: %d\n",
 548                                  max_nob);
 549                         mask |= D_ERROR;
 550                         cfs_trace_put_tcd(tcd);
 551                         tcd = NULL;
 552                         goto console;
 553                 }
 554
 555                 va_start(ap, format);
 556                 if (retry)
 557                         needed = vscnprintf(string_buf, max_nob, format, ap);
 558                 else
 559                         needed = vsnprintf(string_buf, max_nob, format, ap);
 560                 va_end(ap);
 561
 562                 if (needed < max_nob) /* well. printing ok.. */
 563                         break;
 564         }
 565
 566         /* `needed` is actual bytes written to string_buf */
 567         if (*(string_buf + needed - 1) != '\n') {
 568                 pr_info("Lustre: format at %s:%d:%s doesn't end in newline\n",
 569                         file, msgdata->msg_line, msgdata->msg_fn);
 570         } else if (mask & D_TTY) {
 571                 /* TTY needs '\r\n' to move carriage to leftmost position */
 572                 if (needed < 2 || *(string_buf + needed - 2) != '\r')
 573                         pr_info("Lustre: format at %s:%d:%s doesn't end in '\\r\\n'\n",
 574                                 file, msgdata->msg_line, msgdata->msg_fn);
 575         }
 576
 577         header.ph_len = known_size + needed;
 578         debug_buf = (char *)page_address(tage->page) + tage->used;
 579
 580         if (libcfs_debug_binary) {
 581                 memcpy(debug_buf, &header, sizeof(header));
 582                 tage->used += sizeof(header);
 583                 debug_buf += sizeof(header);
 584         }
 585
 586         strlcpy(debug_buf, file, PAGE_SIZE - tage->used);
 587         tage->used += strlen(file) + 1;
 588         debug_buf += strlen(file) + 1;
 589
 590         if (msgdata->msg_fn) {
 591                 strlcpy(debug_buf, msgdata->msg_fn, PAGE_SIZE - tage->used);
 592                 tage->used += strlen(msgdata->msg_fn) + 1;
 593                 debug_buf += strlen(msgdata->msg_fn) + 1;
 594         }
 595
 596         __LASSERT(debug_buf == string_buf);
 597
 598         tage->used += needed;
 599         __LASSERT(tage->used <= PAGE_SIZE);
 600
 601 console:
 602         if ((mask & libcfs_printk) == 0) {
 603                 /* no console output requested */
 604                 if (tcd != NULL)
 605                         cfs_trace_put_tcd(tcd);
 606                 return 1;
 607         }
 608
 609         if (cdls != NULL) {
 610                 if (libcfs_console_ratelimit &&
 611                     cdls->cdls_next != 0 &&     /* not first time ever */
 612                     time_before(jiffies, cdls->cdls_next)) {
 613                         /* skipping a console message */
 614                         cdls->cdls_count++;
 615                         if (tcd != NULL)
 616                                 cfs_trace_put_tcd(tcd);
 617                         return 1;
 618                 }
 619
 620                 if (time_after(jiffies, cdls->cdls_next +
 621                                         libcfs_console_max_delay +
 622                                         cfs_time_seconds(10))) {
 623                         /* last timeout was a long time ago */
 624                         cdls->cdls_delay /= libcfs_console_backoff * 4;
 625                 } else {
 626                         cdls->cdls_delay *= libcfs_console_backoff;
 627                 }
 628
 629                 if (cdls->cdls_delay < libcfs_console_min_delay)
 630                         cdls->cdls_delay = libcfs_console_min_delay;
 631                 else if (cdls->cdls_delay > libcfs_console_max_delay)
 632                         cdls->cdls_delay = libcfs_console_max_delay;
 633
 634                 /* ensure cdls_next is never zero after it's been seen */
 635                 cdls->cdls_next = (jiffies + cdls->cdls_delay) | 1;
 636         }
 637
 638         if (tcd) {
 639                 cfs_print_to_console(&header, mask, string_buf, needed, file,
 640                                      msgdata->msg_fn);
 641                 cfs_trace_put_tcd(tcd);
 642         } else {
 643                 string_buf = cfs_trace_get_console_buffer();
 644
 645                 va_start(ap, format);
 646                 needed = vscnprintf(string_buf, CFS_TRACE_CONSOLE_BUFFER_SIZE,
 647                                     format, ap);
 648                 va_end(ap);
 649
 650                 cfs_print_to_console(&header, mask,
 651                                      string_buf, needed, file, msgdata->msg_fn);
 652
 653                 put_cpu();
 654         }
 655
 656         if (cdls != NULL && cdls->cdls_count != 0) {
 657                 string_buf = cfs_trace_get_console_buffer();
 658
 659                 needed = scnprintf(string_buf, CFS_TRACE_CONSOLE_BUFFER_SIZE,
 660                                    "Skipped %d previous similar message%s\n",
 661                                    cdls->cdls_count,
 662                                    (cdls->cdls_count > 1) ? "s" : "");
 663
 664                 /* Do not allow print this to TTY */
 665                 cfs_print_to_console(&header, mask & ~D_TTY, string_buf,
 666                                      needed, file, msgdata->msg_fn);
 667
 668                 put_cpu();
 669                 cdls->cdls_count = 0;
 670         }
 671
 672         return 0;
 673 }
 674 EXPORT_SYMBOL(libcfs_debug_msg);
 675
 676 void
 677 cfs_trace_assertion_failed(const char *str,
 678                            struct libcfs_debug_msg_data *msgdata)
 679 {
 680         struct ptldebug_header hdr;
 681
 682         libcfs_panic_in_progress = 1;
 683         libcfs_catastrophe = 1;
 684         smp_mb();
 685
 686         cfs_set_ptldebug_header(&hdr, msgdata, CDEBUG_STACK());
 687
 688         cfs_print_to_console(&hdr, D_EMERG, str, strlen(str),
 689                              msgdata->msg_file, msgdata->msg_fn);
 690
 691         panic("Lustre debug assertion failure\n");
 692
 693         /* not reached */
 694 }
 695
 696 static void
 697 panic_collect_pages(struct page_collection *pc)
 698 {
 699         /* Do the collect_pages job on a single CPU: assumes that all other
 700          * CPUs have been stopped during a panic.  If this isn't true for some
 701          * arch, this will have to be implemented separately in each arch.  */
 702         int                        i;
 703         int                        j;
 704         struct cfs_trace_cpu_data *tcd;
 705
 706         INIT_LIST_HEAD(&pc->pc_pages);
 707
 708         cfs_tcd_for_each(tcd, i, j) {
 709                 list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
 710                 tcd->tcd_cur_pages = 0;
 711
 712                 if (pc->pc_want_daemon_pages) {
 713                         list_splice_init(&tcd->tcd_daemon_pages,
 714                                                 &pc->pc_pages);
 715                         tcd->tcd_cur_daemon_pages = 0;
 716                 }
 717         }
 718 }
 719
 720 static void collect_pages_on_all_cpus(struct page_collection *pc)
 721 {
 722         struct cfs_trace_cpu_data *tcd;
 723         int i, cpu;
 724
 725         for_each_possible_cpu(cpu) {
 726                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
 727                         list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
 728                         tcd->tcd_cur_pages = 0;
 729                         if (pc->pc_want_daemon_pages) {
 730                                 list_splice_init(&tcd->tcd_daemon_pages,
 731                                                         &pc->pc_pages);
 732                                 tcd->tcd_cur_daemon_pages = 0;
 733                         }
 734                 }
 735         }
 736 }
 737
 738 static void collect_pages(struct page_collection *pc)
 739 {
 740         INIT_LIST_HEAD(&pc->pc_pages);
 741
 742         if (libcfs_panic_in_progress)
 743                 panic_collect_pages(pc);
 744         else
 745                 collect_pages_on_all_cpus(pc);
 746 }
 747
 748 static void put_pages_back_on_all_cpus(struct page_collection *pc)
 749 {
 750         struct cfs_trace_cpu_data *tcd;
 751         struct list_head *cur_head;
 752         struct cfs_trace_page *tage;
 753         struct cfs_trace_page *tmp;
 754         int i, cpu;
 755
 756         for_each_possible_cpu(cpu) {
 757                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
 758                         cur_head = tcd->tcd_pages.next;
 759
 760                         list_for_each_entry_safe(tage, tmp, &pc->pc_pages,
 761                                                  linkage) {
 762
 763                                 __LASSERT_TAGE_INVARIANT(tage);
 764
 765                                 if (tage->cpu != cpu || tage->type != i)
 766                                         continue;
 767
 768                                 cfs_tage_to_tail(tage, cur_head);
 769                                 tcd->tcd_cur_pages++;
 770                         }
 771                 }
 772         }
 773 }
 774
 775 static void put_pages_back(struct page_collection *pc)
 776 {
 777         if (!libcfs_panic_in_progress)
 778                 put_pages_back_on_all_cpus(pc);
 779 }
 780
 781 /* Add pages to a per-cpu debug daemon ringbuffer.  This buffer makes sure that
 782  * we have a good amount of data at all times for dumping during an LBUG, even
 783  * if we have been steadily writing (and otherwise discarding) pages via the
 784  * debug daemon. */
 785 static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
 786                                          struct cfs_trace_cpu_data *tcd)
 787 {
 788         struct cfs_trace_page *tage;
 789         struct cfs_trace_page *tmp;
 790
 791         list_for_each_entry_safe(tage, tmp, &pc->pc_pages, linkage) {
 792                 __LASSERT_TAGE_INVARIANT(tage);
 793
 794                 if (tage->cpu != tcd->tcd_cpu || tage->type != tcd->tcd_type)
 795                         continue;
 796
 797                 cfs_tage_to_tail(tage, &tcd->tcd_daemon_pages);
 798                 tcd->tcd_cur_daemon_pages++;
 799
 800                 if (tcd->tcd_cur_daemon_pages > tcd->tcd_max_pages) {
 801                         struct cfs_trace_page *victim;
 802
 803                         __LASSERT(!list_empty(&tcd->tcd_daemon_pages));
 804                         victim = cfs_tage_from_list(tcd->tcd_daemon_pages.next);
 805
 806                         __LASSERT_TAGE_INVARIANT(victim);
 807
 808                         list_del(&victim->linkage);
 809                         cfs_tage_free(victim);
 810                         tcd->tcd_cur_daemon_pages--;
 811                 }
 812         }
 813 }
 814
 815 static void put_pages_on_daemon_list(struct page_collection *pc)
 816 {
 817         struct cfs_trace_cpu_data *tcd;
 818         int i, cpu;
 819
 820         for_each_possible_cpu(cpu) {
 821                 cfs_tcd_for_each_type_lock(tcd, i, cpu)
 822                         put_pages_on_tcd_daemon_list(pc, tcd);
 823         }
 824 }
 825
 826 void cfs_trace_debug_print(void)
 827 {
 828         struct page_collection pc;
 829         struct cfs_trace_page *tage;
 830         struct cfs_trace_page *tmp;
 831
 832         pc.pc_want_daemon_pages = 1;
 833         collect_pages(&pc);
 834         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 835                 char *p, *file, *fn;
 836                 struct page *page;
 837
 838                 __LASSERT_TAGE_INVARIANT(tage);
 839
 840                 page = tage->page;
 841                 p = page_address(page);
 842                 while (p < ((char *)page_address(page) + tage->used)) {
 843                         struct ptldebug_header *hdr;
 844                         int len;
 845                         hdr = (void *)p;
 846                         p += sizeof(*hdr);
 847                         file = p;
 848                         p += strlen(file) + 1;
 849                         fn = p;
 850                         p += strlen(fn) + 1;
 851                         len = hdr->ph_len - (int)(p - (char *)hdr);
 852
 853                         cfs_print_to_console(hdr, D_EMERG, p, len, file, fn);
 854
 855                         p += len;
 856                 }
 857
 858                 list_del(&tage->linkage);
 859                 cfs_tage_free(tage);
 860         }
 861 }
 862
 863 int cfs_tracefile_dump_all_pages(char *filename)
 864 {
 865         struct page_collection  pc;
 866         struct file             *filp;
 867         struct cfs_trace_page   *tage;
 868         struct cfs_trace_page   *tmp;
 869         char                    *buf;
 870         int rc;
 871
 872         down_write(&cfs_tracefile_sem);
 873
 874         filp = filp_open(filename, O_CREAT|O_EXCL|O_WRONLY|O_LARGEFILE, 0600);
 875         if (IS_ERR(filp)) {
 876                 rc = PTR_ERR(filp);
 877                 filp = NULL;
 878                 pr_err("LustreError: can't open %s for dump: rc = %d\n",
 879                       filename, rc);
 880                 goto out;
 881         }
 882
 883         pc.pc_want_daemon_pages = 1;
 884         collect_pages(&pc);
 885         if (list_empty(&pc.pc_pages)) {
 886                 rc = 0;
 887                 goto close;
 888         }
 889
 890         /* ok, for now, just write the pages.  in the future we'll be building
 891          * iobufs with the pages and calling generic_direct_IO */
 892         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 893
 894                 __LASSERT_TAGE_INVARIANT(tage);
 895
 896                 buf = kmap(tage->page);
 897                 rc = cfs_kernel_write(filp, buf, tage->used, &filp->f_pos);
 898                 kunmap(tage->page);
 899                 if (rc != (int)tage->used) {
 900                         pr_warn("Lustre: wanted to write %u but wrote %d\n",
 901                                 tage->used, rc);
 902                         put_pages_back(&pc);
 903                         __LASSERT(list_empty(&pc.pc_pages));
 904                         break;
 905                 }
 906                 list_del(&tage->linkage);
 907                 cfs_tage_free(tage);
 908         }
 909
 910         rc = vfs_fsync_range(filp, 0, LLONG_MAX, 1);
 911         if (rc)
 912                 pr_err("LustreError: sync returns: rc = %d\n", rc);
 913 close:
 914         filp_close(filp, NULL);
 915 out:
 916         up_write(&cfs_tracefile_sem);
 917         return rc;
 918 }
 919
 920 void cfs_trace_flush_pages(void)
 921 {
 922         struct page_collection pc;
 923         struct cfs_trace_page *tage;
 924         struct cfs_trace_page *tmp;
 925
 926         pc.pc_want_daemon_pages = 1;
 927         collect_pages(&pc);
 928         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 929
 930                 __LASSERT_TAGE_INVARIANT(tage);
 931
 932                 list_del(&tage->linkage);
 933                 cfs_tage_free(tage);
 934         }
 935 }
 936
 937 int cfs_trace_copyin_string(char *knl_buffer, int knl_buffer_nob,
 938                             const char __user *usr_buffer, int usr_buffer_nob)
 939 {
 940         int    nob;
 941
 942         if (usr_buffer_nob > knl_buffer_nob)
 943                 return -EOVERFLOW;
 944
 945         if (copy_from_user(knl_buffer, usr_buffer, usr_buffer_nob))
 946                 return -EFAULT;
 947
 948         nob = strnlen(knl_buffer, usr_buffer_nob);
 949         while (--nob >= 0)                      /* strip trailing whitespace */
 950                 if (!isspace(knl_buffer[nob]))
 951                         break;
 952
 953         if (nob < 0)                            /* empty string */
 954                 return -EINVAL;
 955
 956         if (nob == knl_buffer_nob)              /* no space to terminate */
 957                 return -EOVERFLOW;
 958
 959         knl_buffer[nob + 1] = 0;                /* terminate */
 960         return 0;
 961 }
 962 EXPORT_SYMBOL(cfs_trace_copyin_string);
 963
 964 int cfs_trace_copyout_string(char __user *usr_buffer, int usr_buffer_nob,
 965                              const char *knl_buffer, char *append)
 966 {
 967         /* NB if 'append' != NULL, it's a single character to append to the
 968          * copied out string - usually "\n", for /proc entries and "" (i.e. a
 969          * terminating zero byte) for sysctl entries */
 970         int   nob = strlen(knl_buffer);
 971
 972         if (nob > usr_buffer_nob)
 973                 nob = usr_buffer_nob;
 974
 975         if (copy_to_user(usr_buffer, knl_buffer, nob))
 976                 return -EFAULT;
 977
 978         if (append != NULL && nob < usr_buffer_nob) {
 979                 if (copy_to_user(usr_buffer + nob, append, 1))
 980                         return -EFAULT;
 981
 982                 nob++;
 983         }
 984
 985         return nob;
 986 }
 987 EXPORT_SYMBOL(cfs_trace_copyout_string);
 988
 989 int cfs_trace_allocate_string_buffer(char **str, int nob)
 990 {
 991         if (nob > 2 * PAGE_SIZE)        /* string must be "sensible" */
 992                 return -EINVAL;
 993
 994         *str = kmalloc(nob, GFP_KERNEL | __GFP_ZERO);
 995         if (*str == NULL)
 996                 return -ENOMEM;
 997
 998         return 0;
 999 }
1000
1001 int cfs_trace_dump_debug_buffer_usrstr(void __user *usr_str, int usr_str_nob)
1002 {
1003         char         *str;
1004         int           rc;
1005
1006         rc = cfs_trace_allocate_string_buffer(&str, usr_str_nob + 1);
1007         if (rc != 0)
1008                 return rc;
1009
1010         rc = cfs_trace_copyin_string(str, usr_str_nob + 1,
1011                                      usr_str, usr_str_nob);
1012         if (rc != 0)
1013                 goto out;
1014
1015         if (str[0] != '/') {
1016                 rc = -EINVAL;
1017                 goto out;
1018         }
1019         rc = cfs_tracefile_dump_all_pages(str);
1020 out:
1021         kfree(str);
1022         return rc;
1023 }
1024
1025 int cfs_trace_daemon_command(char *str)
1026 {
1027         int       rc = 0;
1028
1029         down_write(&cfs_tracefile_sem);
1030
1031         if (strcmp(str, "stop") == 0) {
1032                 up_write(&cfs_tracefile_sem);
1033                 cfs_trace_stop_thread();
1034                 down_write(&cfs_tracefile_sem);
1035                 memset(cfs_tracefile, 0, sizeof(cfs_tracefile));
1036
1037         } else if (strncmp(str, "size=", 5) == 0) {
1038                 unsigned long tmp;
1039
1040                 rc = kstrtoul(str + 5, 10, &tmp);
1041                 if (!rc) {
1042                         if (tmp < 10 || tmp > 20480)
1043                                 cfs_tracefile_size = CFS_TRACEFILE_SIZE;
1044                         else
1045                                 cfs_tracefile_size = tmp << 20;
1046                 }
1047         } else if (strlen(str) >= sizeof(cfs_tracefile)) {
1048                 rc = -ENAMETOOLONG;
1049         } else if (str[0] != '/') {
1050                 rc = -EINVAL;
1051         } else {
1052                 strcpy(cfs_tracefile, str);
1053
1054                 pr_info("Lustre: debug daemon will attempt to start writing to %s (%lukB max)\n",
1055                         cfs_tracefile, (long)(cfs_tracefile_size >> 10));
1056
1057                 cfs_trace_start_thread();
1058         }
1059
1060         up_write(&cfs_tracefile_sem);
1061         return rc;
1062 }
1063
1064 int cfs_trace_daemon_command_usrstr(void __user *usr_str, int usr_str_nob)
1065 {
1066         char *str;
1067         int   rc;
1068
1069         rc = cfs_trace_allocate_string_buffer(&str, usr_str_nob + 1);
1070         if (rc != 0)
1071                 return rc;
1072
1073         rc = cfs_trace_copyin_string(str, usr_str_nob + 1,
1074                                  usr_str, usr_str_nob);
1075         if (rc == 0)
1076                 rc = cfs_trace_daemon_command(str);
1077
1078         kfree(str);
1079         return rc;
1080 }
1081
1082 int cfs_trace_set_debug_mb(int mb)
1083 {
1084         int i;
1085         int j;
1086         unsigned long pages;
1087         unsigned long total_mb = (cfs_totalram_pages() >> (20 - PAGE_SHIFT));
1088         unsigned long limit = max_t(unsigned long, 512, (total_mb * 4) / 5);
1089         struct cfs_trace_cpu_data *tcd;
1090
1091         if (mb < num_possible_cpus()) {
1092                 pr_warn("Lustre: %d MB is too small for debug buffer size, setting it to %d MB.\n",
1093                         mb, num_possible_cpus());
1094                 mb = num_possible_cpus();
1095         }
1096
1097         if (mb > limit) {
1098                 pr_warn("Lustre: %d MB is too large for debug buffer size, setting it to %lu MB.\n",
1099                         mb, limit);
1100                 mb = limit;
1101         }
1102
1103         mb /= num_possible_cpus();
1104         pages = mb << (20 - PAGE_SHIFT);
1105
1106         down_write(&cfs_tracefile_sem);
1107
1108         cfs_tcd_for_each(tcd, i, j)
1109                 tcd->tcd_max_pages = (pages * tcd->tcd_pages_factor) / 100;
1110
1111         up_write(&cfs_tracefile_sem);
1112
1113         return mb;
1114 }
1115
1116 int cfs_trace_get_debug_mb(void)
1117 {
1118         int i;
1119         int j;
1120         struct cfs_trace_cpu_data *tcd;
1121         int total_pages = 0;
1122
1123         down_read(&cfs_tracefile_sem);
1124
1125         cfs_tcd_for_each(tcd, i, j)
1126                 total_pages += tcd->tcd_max_pages;
1127
1128         up_read(&cfs_tracefile_sem);
1129
1130         return (total_pages >> (20 - PAGE_SHIFT)) + 1;
1131 }
1132
1133 static int tracefiled(void *arg)
1134 {
1135         struct page_collection pc;
1136         struct tracefiled_ctl *tctl = arg;
1137         struct cfs_trace_page *tage;
1138         struct cfs_trace_page *tmp;
1139         struct file *filp;
1140         char *buf;
1141         int last_loop = 0;
1142         int rc;
1143
1144         /* we're started late enough that we pick up init's fs context */
1145         /* this is so broken in uml?  what on earth is going on? */
1146
1147         complete(&tctl->tctl_start);
1148
1149         while (1) {
1150                 wait_queue_entry_t __wait;
1151
1152                 pc.pc_want_daemon_pages = 0;
1153                 collect_pages(&pc);
1154                 if (list_empty(&pc.pc_pages))
1155                         goto end_loop;
1156
1157                 filp = NULL;
1158                 down_read(&cfs_tracefile_sem);
1159                 if (cfs_tracefile[0] != 0) {
1160                         filp = filp_open(cfs_tracefile,
1161                                          O_CREAT | O_RDWR | O_LARGEFILE,
1162                                          0600);
1163                         if (IS_ERR(filp)) {
1164                                 rc = PTR_ERR(filp);
1165                                 filp = NULL;
1166                                 pr_warn("Lustre: couldn't open %s: rc = %d\n",
1167                                         cfs_tracefile, rc);
1168                         }
1169                 }
1170                 up_read(&cfs_tracefile_sem);
1171                 if (filp == NULL) {
1172                         put_pages_on_daemon_list(&pc);
1173                         __LASSERT(list_empty(&pc.pc_pages));
1174                         goto end_loop;
1175                 }
1176
1177                 list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
1178                         struct dentry *de = file_dentry(filp);
1179                         static loff_t f_pos;
1180
1181                         __LASSERT_TAGE_INVARIANT(tage);
1182
1183                         if (f_pos >= (off_t)cfs_tracefile_size)
1184                                 f_pos = 0;
1185                         else if (f_pos > i_size_read(de->d_inode))
1186                                 f_pos = i_size_read(de->d_inode);
1187
1188                         buf = kmap(tage->page);
1189                         rc = cfs_kernel_write(filp, buf, tage->used, &f_pos);
1190                         kunmap(tage->page);
1191                         if (rc != (int)tage->used) {
1192                                 pr_warn("Lustre: wanted to write %u but wrote %d\n",
1193                                         tage->used, rc);
1194                                 put_pages_back(&pc);
1195                                 __LASSERT(list_empty(&pc.pc_pages));
1196                                 break;
1197                         }
1198                 }
1199
1200                 filp_close(filp, NULL);
1201                 put_pages_on_daemon_list(&pc);
1202                 if (!list_empty(&pc.pc_pages)) {
1203                         int i;
1204
1205                         pr_alert("Lustre: trace pages aren't empty\n");
1206                         pr_err("Lustre: total cpus(%d): ", num_possible_cpus());
1207                         for (i = 0; i < num_possible_cpus(); i++)
1208                                 if (cpu_online(i))
1209                                         pr_cont("%d(on) ", i);
1210                                 else
1211                                         pr_cont("%d(off) ", i);
1212                         pr_cont("\n");
1213
1214                         i = 0;
1215                         list_for_each_entry_safe(tage, tmp, &pc.pc_pages,
1216                                                  linkage)
1217                                 pr_err("Lustre: page %d belongs to cpu %d\n",
1218                                        ++i, tage->cpu);
1219                         pr_err("Lustre: There are %d pages unwritten\n", i);
1220                 }
1221                 __LASSERT(list_empty(&pc.pc_pages));
1222 end_loop:
1223                 if (atomic_read(&tctl->tctl_shutdown)) {
1224                         if (last_loop == 0) {
1225                                 last_loop = 1;
1226                                 continue;
1227                         } else {
1228                                 break;
1229                         }
1230                 }
1231                 init_waitqueue_entry(&__wait, current);
1232                 add_wait_queue(&tctl->tctl_waitq, &__wait);
1233                 schedule_timeout_interruptible(cfs_time_seconds(1));
1234                 remove_wait_queue(&tctl->tctl_waitq, &__wait);
1235         }
1236         complete(&tctl->tctl_stop);
1237         return 0;
1238 }
1239
1240 int cfs_trace_start_thread(void)
1241 {
1242         struct tracefiled_ctl *tctl = &trace_tctl;
1243         int rc = 0;
1244
1245         mutex_lock(&cfs_trace_thread_mutex);
1246         if (thread_running)
1247                 goto out;
1248
1249         init_completion(&tctl->tctl_start);
1250         init_completion(&tctl->tctl_stop);
1251         init_waitqueue_head(&tctl->tctl_waitq);
1252         atomic_set(&tctl->tctl_shutdown, 0);
1253
1254         if (IS_ERR(kthread_run(tracefiled, tctl, "ktracefiled"))) {
1255                 rc = -ECHILD;
1256                 goto out;
1257         }
1258
1259         wait_for_completion(&tctl->tctl_start);
1260         thread_running = 1;
1261 out:
1262         mutex_unlock(&cfs_trace_thread_mutex);
1263         return rc;
1264 }
1265
1266 void cfs_trace_stop_thread(void)
1267 {
1268         struct tracefiled_ctl *tctl = &trace_tctl;
1269
1270         mutex_lock(&cfs_trace_thread_mutex);
1271         if (thread_running) {
1272                 pr_info("Lustre: shutting down debug daemon thread...\n");
1273                 atomic_set(&tctl->tctl_shutdown, 1);
1274                 wait_for_completion(&tctl->tctl_stop);
1275                 thread_running = 0;
1276         }
1277         mutex_unlock(&cfs_trace_thread_mutex);
1278 }
1279
1280 int cfs_tracefile_init(int max_pages)
1281 {
1282         struct cfs_trace_cpu_data *tcd;
1283         int     i;
1284         int     j;
1285         int     rc;
1286         int     factor;
1287
1288         rc = cfs_tracefile_init_arch();
1289         if (rc != 0)
1290                 return rc;
1291
1292         cfs_tcd_for_each(tcd, i, j) {
1293                 /* tcd_pages_factor is initialized int tracefile_init_arch. */
1294                 factor = tcd->tcd_pages_factor;
1295                 INIT_LIST_HEAD(&tcd->tcd_pages);
1296                 INIT_LIST_HEAD(&tcd->tcd_stock_pages);
1297                 INIT_LIST_HEAD(&tcd->tcd_daemon_pages);
1298                 tcd->tcd_cur_pages = 0;
1299                 tcd->tcd_cur_stock_pages = 0;
1300                 tcd->tcd_cur_daemon_pages = 0;
1301                 tcd->tcd_max_pages = (max_pages * factor) / 100;
1302                 LASSERT(tcd->tcd_max_pages > 0);
1303                 tcd->tcd_shutting_down = 0;
1304         }
1305         return 0;
1306 }
1307
1308 static void trace_cleanup_on_all_cpus(void)
1309 {
1310         struct cfs_trace_cpu_data *tcd;
1311         struct cfs_trace_page *tage;
1312         struct cfs_trace_page *tmp;
1313         int i, cpu;
1314
1315         for_each_possible_cpu(cpu) {
1316                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
1317                         tcd->tcd_shutting_down = 1;
1318
1319                         list_for_each_entry_safe(tage, tmp, &tcd->tcd_pages, linkage) {
1320                                 __LASSERT_TAGE_INVARIANT(tage);
1321
1322                                 list_del(&tage->linkage);
1323                                 cfs_tage_free(tage);
1324                         }
1325                         tcd->tcd_cur_pages = 0;
1326                 }
1327         }
1328 }
1329
1330 static void cfs_trace_cleanup(void)
1331 {
1332         struct page_collection pc;
1333
1334         INIT_LIST_HEAD(&pc.pc_pages);
1335
1336         trace_cleanup_on_all_cpus();
1337
1338         cfs_tracefile_fini_arch();
1339 }
1340
1341 void cfs_tracefile_exit(void)
1342 {
1343         cfs_trace_stop_thread();
1344         cfs_trace_cleanup();
1345 }