libcfs/libcfs/tracefile.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * libcfs/libcfs/tracefile.c
  33  *
  34  * Author: Zach Brown <zab@clusterfs.com>
  35  * Author: Phil Schwan <phil@clusterfs.com>
  36  */
  37
  38 #define DEBUG_SUBSYSTEM S_LNET
  39 #include "tracefile.h"
  40
  41 #include <linux/ctype.h>
  42 #include <linux/fs.h>
  43 #include <linux/kthread.h>
  44 #include <linux/pagemap.h>
  45 #include <linux/poll.h>
  46 #include <linux/tty.h>
  47 #include <linux/uaccess.h>
  48 #include <libcfs/linux/linux-fs.h>
  49 #include <libcfs/libcfs.h>
  50
  51
  52 enum cfs_trace_buf_type {
  53         CFS_TCD_TYPE_PROC = 0,
  54         CFS_TCD_TYPE_SOFTIRQ,
  55         CFS_TCD_TYPE_IRQ,
  56         CFS_TCD_TYPE_CNT
  57 };
  58
  59 union cfs_trace_data_union (*cfs_trace_data[CFS_TCD_TYPE_CNT])[NR_CPUS] __cacheline_aligned;
  60
  61 char cfs_tracefile[TRACEFILE_NAME_SIZE];
  62 long long cfs_tracefile_size = CFS_TRACEFILE_SIZE;
  63 static struct tracefiled_ctl trace_tctl;
  64 static DEFINE_MUTEX(cfs_trace_thread_mutex);
  65 static int thread_running = 0;
  66
  67 static atomic_t cfs_tage_allocated = ATOMIC_INIT(0);
  68 static DECLARE_RWSEM(cfs_tracefile_sem);
  69
  70 static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
  71                                         struct cfs_trace_cpu_data *tcd);
  72
  73 /* trace file lock routines */
  74 /* The walking argument indicates the locking comes from all tcd types
  75  * iterator and we must lock it and dissable local irqs to avoid deadlocks
  76  * with other interrupt locks that might be happening. See LU-1311
  77  * for details.
  78  */
  79 int cfs_trace_lock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
  80         __acquires(&tcd->tcd_lock)
  81 {
  82         __LASSERT(tcd->tcd_type < CFS_TCD_TYPE_CNT);
  83         if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
  84                 spin_lock_irqsave(&tcd->tcd_lock, tcd->tcd_lock_flags);
  85         else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
  86                 spin_lock_bh(&tcd->tcd_lock);
  87         else if (unlikely(walking))
  88                 spin_lock_irq(&tcd->tcd_lock);
  89         else
  90                 spin_lock(&tcd->tcd_lock);
  91         return 1;
  92 }
  93
  94 void cfs_trace_unlock_tcd(struct cfs_trace_cpu_data *tcd, int walking)
  95         __releases(&tcd->tcd_lock)
  96 {
  97         __LASSERT(tcd->tcd_type < CFS_TCD_TYPE_CNT);
  98         if (tcd->tcd_type == CFS_TCD_TYPE_IRQ)
  99                 spin_unlock_irqrestore(&tcd->tcd_lock, tcd->tcd_lock_flags);
 100         else if (tcd->tcd_type == CFS_TCD_TYPE_SOFTIRQ)
 101                 spin_unlock_bh(&tcd->tcd_lock);
 102         else if (unlikely(walking))
 103                 spin_unlock_irq(&tcd->tcd_lock);
 104         else
 105                 spin_unlock(&tcd->tcd_lock);
 106 }
 107
 108 #define cfs_tcd_for_each(tcd, i, j)                                     \
 109         for (i = 0; i < CFS_TCD_TYPE_CNT && cfs_trace_data[i]; i++)     \
 110                 for (j = 0, ((tcd) = &(*cfs_trace_data[i])[j].tcd);     \
 111                      j < num_possible_cpus();                           \
 112                      j++, (tcd) = &(*cfs_trace_data[i])[j].tcd)
 113
 114 #define cfs_tcd_for_each_type_lock(tcd, i, cpu)                         \
 115         for (i = 0; i < CFS_TCD_TYPE_CNT && cfs_trace_data[i] &&        \
 116              (tcd = &(*cfs_trace_data[i])[cpu].tcd) &&                  \
 117              cfs_trace_lock_tcd(tcd, 1); cfs_trace_unlock_tcd(tcd, 1), i++)
 118
 119 enum cfs_trace_buf_type cfs_trace_buf_idx_get(void)
 120 {
 121         if (in_irq())
 122                 return CFS_TCD_TYPE_IRQ;
 123         if (in_softirq())
 124                 return CFS_TCD_TYPE_SOFTIRQ;
 125         return CFS_TCD_TYPE_PROC;
 126 }
 127
 128 static inline struct cfs_trace_cpu_data *
 129 cfs_trace_get_tcd(void)
 130 {
 131         struct cfs_trace_cpu_data *tcd =
 132                 &(*cfs_trace_data[cfs_trace_buf_idx_get()])[get_cpu()].tcd;
 133
 134         cfs_trace_lock_tcd(tcd, 0);
 135
 136         return tcd;
 137 }
 138
 139 static inline void cfs_trace_put_tcd(struct cfs_trace_cpu_data *tcd)
 140 {
 141         cfs_trace_unlock_tcd(tcd, 0);
 142
 143         put_cpu();
 144 }
 145
 146 static inline struct cfs_trace_page *
 147 cfs_tage_from_list(struct list_head *list)
 148 {
 149         return list_entry(list, struct cfs_trace_page, linkage);
 150 }
 151
 152 static struct cfs_trace_page *cfs_tage_alloc(gfp_t gfp)
 153 {
 154         struct page            *page;
 155         struct cfs_trace_page *tage;
 156
 157         /* My caller is trying to free memory */
 158         if (!in_interrupt() && (current->flags & PF_MEMALLOC))
 159                 return NULL;
 160
 161         /*
 162          * Don't spam console with allocation failures: they will be reported
 163          * by upper layer anyway.
 164          */
 165         gfp |= __GFP_NOWARN;
 166         page = alloc_page(gfp);
 167         if (page == NULL)
 168                 return NULL;
 169
 170         tage = kmalloc(sizeof(*tage), gfp);
 171         if (tage == NULL) {
 172                 __free_page(page);
 173                 return NULL;
 174         }
 175
 176         tage->page = page;
 177         atomic_inc(&cfs_tage_allocated);
 178         return tage;
 179 }
 180
 181 static void cfs_tage_free(struct cfs_trace_page *tage)
 182 {
 183         __LASSERT(tage != NULL);
 184         __LASSERT(tage->page != NULL);
 185
 186         __free_page(tage->page);
 187         kfree(tage);
 188         atomic_dec(&cfs_tage_allocated);
 189 }
 190
 191 static void cfs_tage_to_tail(struct cfs_trace_page *tage,
 192                              struct list_head *queue)
 193 {
 194         __LASSERT(tage != NULL);
 195         __LASSERT(queue != NULL);
 196
 197         list_move_tail(&tage->linkage, queue);
 198 }
 199
 200 /* return a page that has 'len' bytes left at the end */
 201 static struct cfs_trace_page *
 202 cfs_trace_get_tage_try(struct cfs_trace_cpu_data *tcd, unsigned long len)
 203 {
 204         struct cfs_trace_page *tage;
 205
 206         if (tcd->tcd_cur_pages > 0) {
 207                 __LASSERT(!list_empty(&tcd->tcd_pages));
 208                 tage = cfs_tage_from_list(tcd->tcd_pages.prev);
 209                 if (tage->used + len <= PAGE_SIZE)
 210                         return tage;
 211         }
 212
 213         if (tcd->tcd_cur_pages < tcd->tcd_max_pages) {
 214                 if (tcd->tcd_cur_stock_pages > 0) {
 215                         tage = cfs_tage_from_list(tcd->tcd_stock_pages.prev);
 216                         --tcd->tcd_cur_stock_pages;
 217                         list_del_init(&tage->linkage);
 218                 } else {
 219                         tage = cfs_tage_alloc(GFP_ATOMIC);
 220                         if (unlikely(tage == NULL)) {
 221                                 if ((!(current->flags & PF_MEMALLOC) ||
 222                                      in_interrupt()) && printk_ratelimit())
 223                                         pr_warn("Lustre: cannot allocate a tage (%ld)\n",
 224                                                 tcd->tcd_cur_pages);
 225                                 return NULL;
 226                         }
 227                 }
 228
 229                 tage->used = 0;
 230                 tage->cpu = smp_processor_id();
 231                 tage->type = tcd->tcd_type;
 232                 list_add_tail(&tage->linkage, &tcd->tcd_pages);
 233                 tcd->tcd_cur_pages++;
 234
 235                 if (tcd->tcd_cur_pages > 8 && thread_running) {
 236                         struct tracefiled_ctl *tctl = &trace_tctl;
 237                         /*
 238                          * wake up tracefiled to process some pages.
 239                          */
 240                         wake_up(&tctl->tctl_waitq);
 241                 }
 242                 return tage;
 243         }
 244         return NULL;
 245 }
 246
 247 static void cfs_tcd_shrink(struct cfs_trace_cpu_data *tcd)
 248 {
 249         int pgcount = tcd->tcd_cur_pages / 10;
 250         struct page_collection pc;
 251         struct cfs_trace_page *tage;
 252         struct cfs_trace_page *tmp;
 253
 254         /*
 255          * XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
 256          * from here: this will lead to infinite recursion.
 257          */
 258
 259         if (printk_ratelimit())
 260                 pr_warn("Lustre: debug daemon buffer overflowed; discarding 10%% of pages (%d of %ld)\n",
 261                         pgcount + 1, tcd->tcd_cur_pages);
 262
 263         INIT_LIST_HEAD(&pc.pc_pages);
 264
 265         list_for_each_entry_safe(tage, tmp, &tcd->tcd_pages, linkage) {
 266                 if (pgcount-- == 0)
 267                         break;
 268
 269                 list_move_tail(&tage->linkage, &pc.pc_pages);
 270                 tcd->tcd_cur_pages--;
 271         }
 272         put_pages_on_tcd_daemon_list(&pc, tcd);
 273 }
 274
 275 /* return a page that has 'len' bytes left at the end */
 276 static struct cfs_trace_page *cfs_trace_get_tage(struct cfs_trace_cpu_data *tcd,
 277                                                  unsigned long len)
 278 {
 279         struct cfs_trace_page *tage;
 280
 281         /*
 282          * XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
 283          * from here: this will lead to infinite recursion.
 284          */
 285
 286         if (len > PAGE_SIZE) {
 287                 pr_err("LustreError: cowardly refusing to write %lu bytes in a page\n",
 288                        len);
 289                 return NULL;
 290         }
 291
 292         tage = cfs_trace_get_tage_try(tcd, len);
 293         if (tage != NULL)
 294                 return tage;
 295         if (thread_running)
 296                 cfs_tcd_shrink(tcd);
 297         if (tcd->tcd_cur_pages > 0) {
 298                 tage = cfs_tage_from_list(tcd->tcd_pages.next);
 299                 tage->used = 0;
 300                 cfs_tage_to_tail(tage, &tcd->tcd_pages);
 301         }
 302         return tage;
 303 }
 304
 305 static void cfs_set_ptldebug_header(struct ptldebug_header *header,
 306                                     struct libcfs_debug_msg_data *msgdata,
 307                                     unsigned long stack)
 308 {
 309         struct timespec64 ts;
 310
 311         ktime_get_real_ts64(&ts);
 312
 313         header->ph_subsys = msgdata->msg_subsys;
 314         header->ph_mask = msgdata->msg_mask;
 315         header->ph_cpu_id = smp_processor_id();
 316         header->ph_type = cfs_trace_buf_idx_get();
 317         /* y2038 safe since all user space treats this as unsigned, but
 318          * will overflow in 2106
 319          */
 320         header->ph_sec = (u32)ts.tv_sec;
 321         header->ph_usec = ts.tv_nsec / NSEC_PER_USEC;
 322         header->ph_stack = stack;
 323         header->ph_pid = current->pid;
 324         header->ph_line_num = msgdata->msg_line;
 325         header->ph_extern_pid = 0;
 326 }
 327
 328 /**
 329  * tty_write_msg - write a message to a certain tty, not just the console.
 330  * @tty: the destination tty_struct
 331  * @msg: the message to write
 332  *
 333  * tty_write_message is not exported, so write a same function for it
 334  *
 335  */
 336 static void tty_write_msg(struct tty_struct *tty, const char *msg)
 337 {
 338         mutex_lock(&tty->atomic_write_lock);
 339         tty_lock(tty);
 340         if (tty->ops->write && tty->count > 0)
 341                 tty->ops->write(tty, msg, strlen(msg));
 342         tty_unlock(tty);
 343         mutex_unlock(&tty->atomic_write_lock);
 344         wake_up_interruptible_poll(&tty->write_wait, POLLOUT);
 345 }
 346
 347 static void cfs_tty_write_message(const char *prefix, int mask, const char *msg)
 348 {
 349         struct tty_struct *tty;
 350
 351         tty = get_current_tty();
 352         if (!tty)
 353                 return;
 354
 355         tty_write_msg(tty, prefix);
 356         if ((mask & D_EMERG) || (mask & D_ERROR))
 357                 tty_write_msg(tty, "Error");
 358         tty_write_msg(tty, ": ");
 359         tty_write_msg(tty, msg);
 360         tty_kref_put(tty);
 361 }
 362
 363 static void cfs_vprint_to_console(struct ptldebug_header *hdr, int mask,
 364                                   struct va_format *vaf, const char *file,
 365                                   const char *fn)
 366 {
 367         char *prefix = "Lustre";
 368
 369         if (hdr->ph_subsys == S_LND || hdr->ph_subsys == S_LNET)
 370                 prefix = "LNet";
 371
 372         if (mask & D_CONSOLE) {
 373                 if (mask & D_EMERG)
 374                         pr_emerg("%sError: %pV", prefix, vaf);
 375                 else if (mask & D_ERROR)
 376                         pr_err("%sError: %pV", prefix, vaf);
 377                 else if (mask & D_WARNING)
 378                         pr_warn("%s: %pV", prefix, vaf);
 379                 else if (mask & libcfs_printk)
 380                         pr_info("%s: %pV", prefix, vaf);
 381         } else {
 382                 if (mask & D_EMERG)
 383                         pr_emerg("%sError: %d:%d:(%s:%d:%s()) %pV", prefix,
 384                                  hdr->ph_pid, hdr->ph_extern_pid, file,
 385                                  hdr->ph_line_num, fn, vaf);
 386                 else if (mask & D_ERROR)
 387                         pr_err("%sError: %d:%d:(%s:%d:%s()) %pV", prefix,
 388                                hdr->ph_pid, hdr->ph_extern_pid, file,
 389                                hdr->ph_line_num, fn, vaf);
 390                 else if (mask & D_WARNING)
 391                         pr_warn("%s: %d:%d:(%s:%d:%s()) %pV", prefix,
 392                                 hdr->ph_pid, hdr->ph_extern_pid, file,
 393                                 hdr->ph_line_num, fn, vaf);
 394                 else if (mask & (D_CONSOLE | libcfs_printk))
 395                         pr_info("%s: %pV", prefix, vaf);
 396         }
 397
 398         if (mask & D_TTY)
 399                 /* tty_write_msg doesn't handle formatting */
 400                 cfs_tty_write_message(prefix, mask, vaf->fmt);
 401 }
 402
 403 static void cfs_print_to_console(struct ptldebug_header *hdr, int mask,
 404                                  const char *file, const char *fn,
 405                                  const char *fmt, ...)
 406 {
 407         struct va_format vaf;
 408         va_list args;
 409
 410         va_start(args, fmt);
 411         vaf.fmt = fmt;
 412         vaf.va = &args;
 413         cfs_vprint_to_console(hdr, mask, &vaf, file, fn);
 414 }
 415
 416 int libcfs_debug_msg(struct libcfs_debug_msg_data *msgdata,
 417                      const char *format, ...)
 418 {
 419         struct cfs_trace_cpu_data *tcd = NULL;
 420         struct ptldebug_header header = {0};
 421         struct cfs_trace_page *tage;
 422         /* string_buf is used only if tcd != NULL, and is always set then */
 423         char *string_buf = NULL;
 424         char *debug_buf;
 425         int known_size;
 426         int needed = 85; /* seeded with average message length */
 427         int max_nob;
 428         va_list ap;
 429         int retry;
 430         int mask = msgdata->msg_mask;
 431         char *file = (char *)msgdata->msg_file;
 432         struct cfs_debug_limit_state *cdls = msgdata->msg_cdls;
 433
 434         if (strchr(file, '/'))
 435                 file = strrchr(file, '/') + 1;
 436
 437         tcd = cfs_trace_get_tcd();
 438
 439         /* cfs_trace_get_tcd() grabs a lock, which disables preemption and
 440          * pins us to a particular CPU.  This avoids an smp_processor_id()
 441          * warning on Linux when debugging is enabled.
 442          */
 443         cfs_set_ptldebug_header(&header, msgdata, CDEBUG_STACK());
 444
 445         if (!tcd)                /* arch may not log in IRQ context */
 446                 goto console;
 447
 448         if (tcd->tcd_cur_pages == 0)
 449                 header.ph_flags |= PH_FLAG_FIRST_RECORD;
 450
 451         if (tcd->tcd_shutting_down) {
 452                 cfs_trace_put_tcd(tcd);
 453                 tcd = NULL;
 454                 goto console;
 455         }
 456
 457         known_size = strlen(file) + 1;
 458         if (msgdata->msg_fn)
 459                 known_size += strlen(msgdata->msg_fn) + 1;
 460
 461         if (libcfs_debug_binary)
 462                 known_size += sizeof(header);
 463
 464         /*
 465          * May perform an additional pass to update 'needed' and increase
 466          * tage buffer size to match vsnprintf reported size required
 467          * On the second pass (retry=1) use vscnprintf [which returns
 468          * number of bytes written not including the terminating nul]
 469          * to clarify `needed` is used as number of bytes written
 470          * for the remainder of this function
 471          */
 472         for (retry = 0; retry < 2; retry++) {
 473                 tage = cfs_trace_get_tage(tcd, needed + known_size + 1);
 474                 if (!tage) {
 475                         if (needed + known_size > PAGE_SIZE)
 476                                 mask |= D_ERROR;
 477
 478                         cfs_trace_put_tcd(tcd);
 479                         tcd = NULL;
 480                         goto console;
 481                 }
 482
 483                 string_buf = (char *)page_address(tage->page) +
 484                              tage->used + known_size;
 485
 486                 max_nob = PAGE_SIZE - tage->used - known_size;
 487                 if (max_nob <= 0) {
 488                         pr_emerg("LustreError: negative max_nob: %d\n",
 489                                  max_nob);
 490                         mask |= D_ERROR;
 491                         cfs_trace_put_tcd(tcd);
 492                         tcd = NULL;
 493                         goto console;
 494                 }
 495
 496                 va_start(ap, format);
 497                 if (retry)
 498                         needed = vscnprintf(string_buf, max_nob, format, ap);
 499                 else
 500                         needed = vsnprintf(string_buf, max_nob, format, ap);
 501                 va_end(ap);
 502
 503                 if (needed < max_nob) /* well. printing ok.. */
 504                         break;
 505         }
 506
 507         /* `needed` is actual bytes written to string_buf */
 508         if (*(string_buf + needed - 1) != '\n') {
 509                 pr_info("Lustre: format at %s:%d:%s doesn't end in newline\n",
 510                         file, msgdata->msg_line, msgdata->msg_fn);
 511         } else if (mask & D_TTY) {
 512                 /* TTY needs '\r\n' to move carriage to leftmost position */
 513                 if (needed < 2 || *(string_buf + needed - 2) != '\r')
 514                         pr_info("Lustre: format at %s:%d:%s doesn't end in '\\r\\n'\n",
 515                                 file, msgdata->msg_line, msgdata->msg_fn);
 516                 if (strnchr(string_buf, needed, '%'))
 517                         pr_info("Lustre: format at %s:%d:%s mustn't contain %%\n",
 518                                 file, msgdata->msg_line, msgdata->msg_fn);
 519         }
 520
 521         header.ph_len = known_size + needed;
 522         debug_buf = (char *)page_address(tage->page) + tage->used;
 523
 524         if (libcfs_debug_binary) {
 525                 memcpy(debug_buf, &header, sizeof(header));
 526                 tage->used += sizeof(header);
 527                 debug_buf += sizeof(header);
 528         }
 529
 530         strlcpy(debug_buf, file, PAGE_SIZE - tage->used);
 531         tage->used += strlen(file) + 1;
 532         debug_buf += strlen(file) + 1;
 533
 534         if (msgdata->msg_fn) {
 535                 strlcpy(debug_buf, msgdata->msg_fn, PAGE_SIZE - tage->used);
 536                 tage->used += strlen(msgdata->msg_fn) + 1;
 537                 debug_buf += strlen(msgdata->msg_fn) + 1;
 538         }
 539
 540         __LASSERT(debug_buf == string_buf);
 541
 542         tage->used += needed;
 543         __LASSERT(tage->used <= PAGE_SIZE);
 544
 545 console:
 546         if ((mask & libcfs_printk) == 0) {
 547                 /* no console output requested */
 548                 if (tcd != NULL)
 549                         cfs_trace_put_tcd(tcd);
 550                 return 1;
 551         }
 552
 553         if (cdls != NULL) {
 554                 if (libcfs_console_ratelimit &&
 555                     cdls->cdls_next != 0 &&     /* not first time ever */
 556                     time_before(jiffies, cdls->cdls_next)) {
 557                         /* skipping a console message */
 558                         cdls->cdls_count++;
 559                         if (tcd != NULL)
 560                                 cfs_trace_put_tcd(tcd);
 561                         return 1;
 562                 }
 563
 564                 if (time_after(jiffies, cdls->cdls_next +
 565                                         libcfs_console_max_delay +
 566                                         cfs_time_seconds(10))) {
 567                         /* last timeout was a long time ago */
 568                         cdls->cdls_delay /= libcfs_console_backoff * 4;
 569                 } else {
 570                         cdls->cdls_delay *= libcfs_console_backoff;
 571                 }
 572
 573                 if (cdls->cdls_delay < libcfs_console_min_delay)
 574                         cdls->cdls_delay = libcfs_console_min_delay;
 575                 else if (cdls->cdls_delay > libcfs_console_max_delay)
 576                         cdls->cdls_delay = libcfs_console_max_delay;
 577
 578                 /* ensure cdls_next is never zero after it's been seen */
 579                 cdls->cdls_next = (jiffies + cdls->cdls_delay) | 1;
 580         }
 581
 582         if (tcd) {
 583                 cfs_print_to_console(&header, mask, file, msgdata->msg_fn,
 584                                      "%s", string_buf);
 585                 cfs_trace_put_tcd(tcd);
 586         } else {
 587                 struct va_format vaf;
 588
 589                 va_start(ap, format);
 590                 vaf.fmt = format;
 591                 vaf.va = &ap;
 592                 cfs_vprint_to_console(&header, mask,
 593                                       &vaf, file, msgdata->msg_fn);
 594                 va_end(ap);
 595         }
 596
 597         if (cdls != NULL && cdls->cdls_count != 0) {
 598                 /* Do not allow print this to TTY */
 599                 cfs_print_to_console(&header, mask & ~D_TTY, file,
 600                                      msgdata->msg_fn,
 601                                      "Skipped %d previous similar message%s\n",
 602                                      cdls->cdls_count,
 603                                      (cdls->cdls_count > 1) ? "s" : "");
 604
 605                 cdls->cdls_count = 0;
 606         }
 607
 608         return 0;
 609 }
 610 EXPORT_SYMBOL(libcfs_debug_msg);
 611
 612 void
 613 cfs_trace_assertion_failed(const char *str,
 614                            struct libcfs_debug_msg_data *msgdata)
 615 {
 616         struct ptldebug_header hdr;
 617
 618         libcfs_panic_in_progress = 1;
 619         libcfs_catastrophe = 1;
 620         smp_mb();
 621
 622         cfs_set_ptldebug_header(&hdr, msgdata, CDEBUG_STACK());
 623
 624         cfs_print_to_console(&hdr, D_EMERG, msgdata->msg_file, msgdata->msg_fn,
 625                              "%s", str);
 626
 627         panic("Lustre debug assertion failure\n");
 628
 629         /* not reached */
 630 }
 631
 632 static void
 633 panic_collect_pages(struct page_collection *pc)
 634 {
 635         /* Do the collect_pages job on a single CPU: assumes that all other
 636          * CPUs have been stopped during a panic.  If this isn't true for some
 637          * arch, this will have to be implemented separately in each arch.  */
 638         int                        i;
 639         int                        j;
 640         struct cfs_trace_cpu_data *tcd;
 641
 642         INIT_LIST_HEAD(&pc->pc_pages);
 643
 644         cfs_tcd_for_each(tcd, i, j) {
 645                 list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
 646                 tcd->tcd_cur_pages = 0;
 647
 648                 if (pc->pc_want_daemon_pages) {
 649                         list_splice_init(&tcd->tcd_daemon_pages,
 650                                                 &pc->pc_pages);
 651                         tcd->tcd_cur_daemon_pages = 0;
 652                 }
 653         }
 654 }
 655
 656 static void collect_pages_on_all_cpus(struct page_collection *pc)
 657 {
 658         struct cfs_trace_cpu_data *tcd;
 659         int i, cpu;
 660
 661         for_each_possible_cpu(cpu) {
 662                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
 663                         list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
 664                         tcd->tcd_cur_pages = 0;
 665                         if (pc->pc_want_daemon_pages) {
 666                                 list_splice_init(&tcd->tcd_daemon_pages,
 667                                                         &pc->pc_pages);
 668                                 tcd->tcd_cur_daemon_pages = 0;
 669                         }
 670                 }
 671         }
 672 }
 673
 674 static void collect_pages(struct page_collection *pc)
 675 {
 676         INIT_LIST_HEAD(&pc->pc_pages);
 677
 678         if (libcfs_panic_in_progress)
 679                 panic_collect_pages(pc);
 680         else
 681                 collect_pages_on_all_cpus(pc);
 682 }
 683
 684 static void put_pages_back_on_all_cpus(struct page_collection *pc)
 685 {
 686         struct cfs_trace_cpu_data *tcd;
 687         struct list_head *cur_head;
 688         struct cfs_trace_page *tage;
 689         struct cfs_trace_page *tmp;
 690         int i, cpu;
 691
 692         for_each_possible_cpu(cpu) {
 693                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
 694                         cur_head = tcd->tcd_pages.next;
 695
 696                         list_for_each_entry_safe(tage, tmp, &pc->pc_pages,
 697                                                  linkage) {
 698
 699                                 __LASSERT_TAGE_INVARIANT(tage);
 700
 701                                 if (tage->cpu != cpu || tage->type != i)
 702                                         continue;
 703
 704                                 cfs_tage_to_tail(tage, cur_head);
 705                                 tcd->tcd_cur_pages++;
 706                         }
 707                 }
 708         }
 709 }
 710
 711 static void put_pages_back(struct page_collection *pc)
 712 {
 713         if (!libcfs_panic_in_progress)
 714                 put_pages_back_on_all_cpus(pc);
 715 }
 716
 717 /* Add pages to a per-cpu debug daemon ringbuffer.  This buffer makes sure that
 718  * we have a good amount of data at all times for dumping during an LBUG, even
 719  * if we have been steadily writing (and otherwise discarding) pages via the
 720  * debug daemon. */
 721 static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
 722                                          struct cfs_trace_cpu_data *tcd)
 723 {
 724         struct cfs_trace_page *tage;
 725         struct cfs_trace_page *tmp;
 726
 727         list_for_each_entry_safe(tage, tmp, &pc->pc_pages, linkage) {
 728                 __LASSERT_TAGE_INVARIANT(tage);
 729
 730                 if (tage->cpu != tcd->tcd_cpu || tage->type != tcd->tcd_type)
 731                         continue;
 732
 733                 cfs_tage_to_tail(tage, &tcd->tcd_daemon_pages);
 734                 tcd->tcd_cur_daemon_pages++;
 735
 736                 if (tcd->tcd_cur_daemon_pages > tcd->tcd_max_pages) {
 737                         struct cfs_trace_page *victim;
 738
 739                         __LASSERT(!list_empty(&tcd->tcd_daemon_pages));
 740                         victim = cfs_tage_from_list(tcd->tcd_daemon_pages.next);
 741
 742                         __LASSERT_TAGE_INVARIANT(victim);
 743
 744                         list_del(&victim->linkage);
 745                         cfs_tage_free(victim);
 746                         tcd->tcd_cur_daemon_pages--;
 747                 }
 748         }
 749 }
 750
 751 static void put_pages_on_daemon_list(struct page_collection *pc)
 752 {
 753         struct cfs_trace_cpu_data *tcd;
 754         int i, cpu;
 755
 756         for_each_possible_cpu(cpu) {
 757                 cfs_tcd_for_each_type_lock(tcd, i, cpu)
 758                         put_pages_on_tcd_daemon_list(pc, tcd);
 759         }
 760 }
 761
 762 #ifdef LNET_DUMP_ON_PANIC
 763 void cfs_trace_debug_print(void)
 764 {
 765         struct page_collection pc;
 766         struct cfs_trace_page *tage;
 767         struct cfs_trace_page *tmp;
 768
 769         pc.pc_want_daemon_pages = 1;
 770         collect_pages(&pc);
 771         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 772                 char *p, *file, *fn;
 773                 struct page *page;
 774
 775                 __LASSERT_TAGE_INVARIANT(tage);
 776
 777                 page = tage->page;
 778                 p = page_address(page);
 779                 while (p < ((char *)page_address(page) + tage->used)) {
 780                         struct ptldebug_header *hdr;
 781                         int len;
 782                         hdr = (void *)p;
 783                         p += sizeof(*hdr);
 784                         file = p;
 785                         p += strlen(file) + 1;
 786                         fn = p;
 787                         p += strlen(fn) + 1;
 788                         len = hdr->ph_len - (int)(p - (char *)hdr);
 789
 790                         cfs_print_to_console(hdr, D_EMERG, file, fn,
 791                                              "%.*s", len, p);
 792
 793                         p += len;
 794                 }
 795
 796                 list_del(&tage->linkage);
 797                 cfs_tage_free(tage);
 798         }
 799 }
 800 #endif /* LNET_DUMP_ON_PANIC */
 801
 802 int cfs_tracefile_dump_all_pages(char *filename)
 803 {
 804         struct page_collection  pc;
 805         struct file             *filp;
 806         struct cfs_trace_page   *tage;
 807         struct cfs_trace_page   *tmp;
 808         char                    *buf;
 809         int rc;
 810
 811         down_write(&cfs_tracefile_sem);
 812
 813         filp = filp_open(filename, O_CREAT|O_EXCL|O_WRONLY|O_LARGEFILE, 0600);
 814         if (IS_ERR(filp)) {
 815                 rc = PTR_ERR(filp);
 816                 filp = NULL;
 817                 pr_err("LustreError: can't open %s for dump: rc = %d\n",
 818                       filename, rc);
 819                 goto out;
 820         }
 821
 822         pc.pc_want_daemon_pages = 1;
 823         collect_pages(&pc);
 824         if (list_empty(&pc.pc_pages)) {
 825                 rc = 0;
 826                 goto close;
 827         }
 828
 829         /* ok, for now, just write the pages.  in the future we'll be building
 830          * iobufs with the pages and calling generic_direct_IO */
 831         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 832
 833                 __LASSERT_TAGE_INVARIANT(tage);
 834
 835                 buf = kmap(tage->page);
 836                 rc = cfs_kernel_write(filp, buf, tage->used, &filp->f_pos);
 837                 kunmap(tage->page);
 838                 if (rc != (int)tage->used) {
 839                         pr_warn("Lustre: wanted to write %u but wrote %d\n",
 840                                 tage->used, rc);
 841                         put_pages_back(&pc);
 842                         __LASSERT(list_empty(&pc.pc_pages));
 843                         break;
 844                 }
 845                 list_del(&tage->linkage);
 846                 cfs_tage_free(tage);
 847         }
 848
 849         rc = vfs_fsync_range(filp, 0, LLONG_MAX, 1);
 850         if (rc)
 851                 pr_err("LustreError: sync returns: rc = %d\n", rc);
 852 close:
 853         filp_close(filp, NULL);
 854 out:
 855         up_write(&cfs_tracefile_sem);
 856         return rc;
 857 }
 858
 859 void cfs_trace_flush_pages(void)
 860 {
 861         struct page_collection pc;
 862         struct cfs_trace_page *tage;
 863
 864         pc.pc_want_daemon_pages = 1;
 865         collect_pages(&pc);
 866         while (!list_empty(&pc.pc_pages)) {
 867                 tage = list_first_entry(&pc.pc_pages,
 868                                         struct cfs_trace_page, linkage);
 869                 __LASSERT_TAGE_INVARIANT(tage);
 870
 871                 list_del(&tage->linkage);
 872                 cfs_tage_free(tage);
 873         }
 874 }
 875
 876 int cfs_trace_copyout_string(char __user *usr_buffer, int usr_buffer_nob,
 877                              const char *knl_buffer, char *append)
 878 {
 879         /* NB if 'append' != NULL, it's a single character to append to the
 880          * copied out string - usually "\n", for /proc entries and "" (i.e. a
 881          * terminating zero byte) for sysctl entries */
 882         int   nob = strlen(knl_buffer);
 883
 884         if (nob > usr_buffer_nob)
 885                 nob = usr_buffer_nob;
 886
 887         if (copy_to_user(usr_buffer, knl_buffer, nob))
 888                 return -EFAULT;
 889
 890         if (append != NULL && nob < usr_buffer_nob) {
 891                 if (copy_to_user(usr_buffer + nob, append, 1))
 892                         return -EFAULT;
 893
 894                 nob++;
 895         }
 896
 897         return nob;
 898 }
 899 EXPORT_SYMBOL(cfs_trace_copyout_string);
 900
 901 int cfs_trace_allocate_string_buffer(char **str, int nob)
 902 {
 903         if (nob > 2 * PAGE_SIZE)        /* string must be "sensible" */
 904                 return -EINVAL;
 905
 906         *str = kmalloc(nob, GFP_KERNEL | __GFP_ZERO);
 907         if (*str == NULL)
 908                 return -ENOMEM;
 909
 910         return 0;
 911 }
 912
 913 int cfs_trace_dump_debug_buffer_usrstr(void __user *usr_str, int usr_str_nob)
 914 {
 915         char *str;
 916         char *path;
 917         int rc;
 918
 919         str = memdup_user_nul(usr_str, usr_str_nob);
 920         if (!str)
 921                 return -ENOMEM;
 922
 923         path = strim(str);
 924         if (path[0] != '/')
 925                 rc = -EINVAL;
 926         else
 927                 rc = cfs_tracefile_dump_all_pages(path);
 928         kfree(str);
 929
 930         return rc;
 931 }
 932
 933 int cfs_trace_daemon_command(char *str)
 934 {
 935         int       rc = 0;
 936
 937         down_write(&cfs_tracefile_sem);
 938
 939         if (strcmp(str, "stop") == 0) {
 940                 up_write(&cfs_tracefile_sem);
 941                 cfs_trace_stop_thread();
 942                 down_write(&cfs_tracefile_sem);
 943                 memset(cfs_tracefile, 0, sizeof(cfs_tracefile));
 944
 945         } else if (strncmp(str, "size=", 5) == 0) {
 946                 unsigned long tmp;
 947
 948                 rc = kstrtoul(str + 5, 10, &tmp);
 949                 if (!rc) {
 950                         if (tmp < 10 || tmp > 20480)
 951                                 cfs_tracefile_size = CFS_TRACEFILE_SIZE;
 952                         else
 953                                 cfs_tracefile_size = tmp << 20;
 954                 }
 955         } else if (strlen(str) >= sizeof(cfs_tracefile)) {
 956                 rc = -ENAMETOOLONG;
 957         } else if (str[0] != '/') {
 958                 rc = -EINVAL;
 959         } else {
 960                 strcpy(cfs_tracefile, str);
 961
 962                 pr_info("Lustre: debug daemon will attempt to start writing to %s (%lukB max)\n",
 963                         cfs_tracefile, (long)(cfs_tracefile_size >> 10));
 964
 965                 cfs_trace_start_thread();
 966         }
 967
 968         up_write(&cfs_tracefile_sem);
 969         return rc;
 970 }
 971
 972 int cfs_trace_daemon_command_usrstr(void __user *usr_str, int usr_str_nob)
 973 {
 974         char *str;
 975         int   rc;
 976
 977         str = memdup_user_nul(usr_str, usr_str_nob);
 978         if (!str)
 979                 return -ENOMEM;
 980
 981         rc = cfs_trace_daemon_command(strim(str));
 982         kfree(str);
 983
 984         return rc;
 985 }
 986
 987 int cfs_trace_set_debug_mb(int mb)
 988 {
 989         int i;
 990         int j;
 991         unsigned long pages;
 992         unsigned long total_mb = (cfs_totalram_pages() >> (20 - PAGE_SHIFT));
 993         unsigned long limit = max_t(unsigned long, 512, (total_mb * 4) / 5);
 994         struct cfs_trace_cpu_data *tcd;
 995
 996         if (mb < num_possible_cpus()) {
 997                 pr_warn("Lustre: %d MB is too small for debug buffer size, setting it to %d MB.\n",
 998                         mb, num_possible_cpus());
 999                 mb = num_possible_cpus();
1000         }
1001
1002         if (mb > limit) {
1003                 pr_warn("Lustre: %d MB is too large for debug buffer size, setting it to %lu MB.\n",
1004                         mb, limit);
1005                 mb = limit;
1006         }
1007
1008         mb /= num_possible_cpus();
1009         pages = mb << (20 - PAGE_SHIFT);
1010
1011         down_write(&cfs_tracefile_sem);
1012
1013         cfs_tcd_for_each(tcd, i, j)
1014                 tcd->tcd_max_pages = (pages * tcd->tcd_pages_factor) / 100;
1015
1016         up_write(&cfs_tracefile_sem);
1017
1018         return mb;
1019 }
1020
1021 int cfs_trace_get_debug_mb(void)
1022 {
1023         int i;
1024         int j;
1025         struct cfs_trace_cpu_data *tcd;
1026         int total_pages = 0;
1027
1028         down_read(&cfs_tracefile_sem);
1029
1030         cfs_tcd_for_each(tcd, i, j)
1031                 total_pages += tcd->tcd_max_pages;
1032
1033         up_read(&cfs_tracefile_sem);
1034
1035         return (total_pages >> (20 - PAGE_SHIFT)) + 1;
1036 }
1037
1038 static int tracefiled(void *arg)
1039 {
1040         struct page_collection pc;
1041         struct tracefiled_ctl *tctl = arg;
1042         struct cfs_trace_page *tage;
1043         struct cfs_trace_page *tmp;
1044         struct file *filp;
1045         char *buf;
1046         int last_loop = 0;
1047         int rc;
1048
1049         /* we're started late enough that we pick up init's fs context */
1050         /* this is so broken in uml?  what on earth is going on? */
1051
1052         complete(&tctl->tctl_start);
1053
1054         pc.pc_want_daemon_pages = 0;
1055
1056         while (!last_loop) {
1057                 wait_event_timeout(tctl->tctl_waitq,
1058                                    ({ collect_pages(&pc);
1059                                      !list_empty(&pc.pc_pages); }) ||
1060                                    atomic_read(&tctl->tctl_shutdown),
1061                                    cfs_time_seconds(1));
1062                 if (atomic_read(&tctl->tctl_shutdown))
1063                         last_loop = 1;
1064                 if (list_empty(&pc.pc_pages))
1065                         continue;
1066
1067                 filp = NULL;
1068                 down_read(&cfs_tracefile_sem);
1069                 if (cfs_tracefile[0] != 0) {
1070                         filp = filp_open(cfs_tracefile,
1071                                          O_CREAT | O_RDWR | O_LARGEFILE,
1072                                          0600);
1073                         if (IS_ERR(filp)) {
1074                                 rc = PTR_ERR(filp);
1075                                 filp = NULL;
1076                                 pr_warn("Lustre: couldn't open %s: rc = %d\n",
1077                                         cfs_tracefile, rc);
1078                         }
1079                 }
1080                 up_read(&cfs_tracefile_sem);
1081                 if (filp == NULL) {
1082                         put_pages_on_daemon_list(&pc);
1083                         __LASSERT(list_empty(&pc.pc_pages));
1084                         continue;
1085                 }
1086
1087                 list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
1088                         struct dentry *de = file_dentry(filp);
1089                         static loff_t f_pos;
1090
1091                         __LASSERT_TAGE_INVARIANT(tage);
1092
1093                         if (f_pos >= (off_t)cfs_tracefile_size)
1094                                 f_pos = 0;
1095                         else if (f_pos > i_size_read(de->d_inode))
1096                                 f_pos = i_size_read(de->d_inode);
1097
1098                         buf = kmap(tage->page);
1099                         rc = cfs_kernel_write(filp, buf, tage->used, &f_pos);
1100                         kunmap(tage->page);
1101                         if (rc != (int)tage->used) {
1102                                 pr_warn("Lustre: wanted to write %u but wrote %d\n",
1103                                         tage->used, rc);
1104                                 put_pages_back(&pc);
1105                                 __LASSERT(list_empty(&pc.pc_pages));
1106                                 break;
1107                         }
1108                 }
1109
1110                 filp_close(filp, NULL);
1111                 put_pages_on_daemon_list(&pc);
1112                 if (!list_empty(&pc.pc_pages)) {
1113                         int i;
1114
1115                         pr_alert("Lustre: trace pages aren't empty\n");
1116                         pr_err("Lustre: total cpus(%d): ", num_possible_cpus());
1117                         for (i = 0; i < num_possible_cpus(); i++)
1118                                 if (cpu_online(i))
1119                                         pr_cont("%d(on) ", i);
1120                                 else
1121                                         pr_cont("%d(off) ", i);
1122                         pr_cont("\n");
1123
1124                         i = 0;
1125                         list_for_each_entry_safe(tage, tmp, &pc.pc_pages,
1126                                                  linkage)
1127                                 pr_err("Lustre: page %d belongs to cpu %d\n",
1128                                        ++i, tage->cpu);
1129                         pr_err("Lustre: There are %d pages unwritten\n", i);
1130                 }
1131                 __LASSERT(list_empty(&pc.pc_pages));
1132         }
1133         complete(&tctl->tctl_stop);
1134         return 0;
1135 }
1136
1137 int cfs_trace_start_thread(void)
1138 {
1139         struct tracefiled_ctl *tctl = &trace_tctl;
1140         int rc = 0;
1141
1142         mutex_lock(&cfs_trace_thread_mutex);
1143         if (thread_running)
1144                 goto out;
1145
1146         init_completion(&tctl->tctl_start);
1147         init_completion(&tctl->tctl_stop);
1148         init_waitqueue_head(&tctl->tctl_waitq);
1149         atomic_set(&tctl->tctl_shutdown, 0);
1150
1151         if (IS_ERR(kthread_run(tracefiled, tctl, "ktracefiled"))) {
1152                 rc = -ECHILD;
1153                 goto out;
1154         }
1155
1156         wait_for_completion(&tctl->tctl_start);
1157         thread_running = 1;
1158 out:
1159         mutex_unlock(&cfs_trace_thread_mutex);
1160         return rc;
1161 }
1162
1163 void cfs_trace_stop_thread(void)
1164 {
1165         struct tracefiled_ctl *tctl = &trace_tctl;
1166
1167         mutex_lock(&cfs_trace_thread_mutex);
1168         if (thread_running) {
1169                 pr_info("Lustre: shutting down debug daemon thread...\n");
1170                 atomic_set(&tctl->tctl_shutdown, 1);
1171                 wait_for_completion(&tctl->tctl_stop);
1172                 thread_running = 0;
1173         }
1174         mutex_unlock(&cfs_trace_thread_mutex);
1175 }
1176
1177 /* percents to share the total debug memory for each type */
1178 static unsigned int pages_factor[CFS_TCD_TYPE_CNT] = {
1179         80, /* 80% pages for CFS_TCD_TYPE_PROC */
1180         10, /* 10% pages for CFS_TCD_TYPE_SOFTIRQ */
1181         10  /* 10% pages for CFS_TCD_TYPE_IRQ */
1182 };
1183
1184 int cfs_tracefile_init(int max_pages)
1185 {
1186         struct cfs_trace_cpu_data *tcd;
1187         int i;
1188         int j;
1189
1190         /* initialize trace_data */
1191         memset(cfs_trace_data, 0, sizeof(cfs_trace_data));
1192         for (i = 0; i < CFS_TCD_TYPE_CNT; i++) {
1193                 cfs_trace_data[i] =
1194                         kmalloc_array(num_possible_cpus(),
1195                                       sizeof(union cfs_trace_data_union),
1196                                       GFP_KERNEL);
1197                 if (!cfs_trace_data[i])
1198                         goto out_trace_data;
1199         }
1200
1201         /* arch related info initialized */
1202         cfs_tcd_for_each(tcd, i, j) {
1203                 int factor = pages_factor[i];
1204
1205                 spin_lock_init(&tcd->tcd_lock);
1206                 tcd->tcd_pages_factor = factor;
1207                 tcd->tcd_type = i;
1208                 tcd->tcd_cpu = j;
1209
1210                 INIT_LIST_HEAD(&tcd->tcd_pages);
1211                 INIT_LIST_HEAD(&tcd->tcd_stock_pages);
1212                 INIT_LIST_HEAD(&tcd->tcd_daemon_pages);
1213                 tcd->tcd_cur_pages = 0;
1214                 tcd->tcd_cur_stock_pages = 0;
1215                 tcd->tcd_cur_daemon_pages = 0;
1216                 tcd->tcd_max_pages = (max_pages * factor) / 100;
1217                 LASSERT(tcd->tcd_max_pages > 0);
1218                 tcd->tcd_shutting_down = 0;
1219         }
1220
1221         return 0;
1222
1223 out_trace_data:
1224         for (i = 0; cfs_trace_data[i]; i++) {
1225                 kfree(cfs_trace_data[i]);
1226                 cfs_trace_data[i] = NULL;
1227         }
1228         pr_err("lnet: Not enough memory\n");
1229         return -ENOMEM;
1230 }
1231
1232 static void trace_cleanup_on_all_cpus(void)
1233 {
1234         struct cfs_trace_cpu_data *tcd;
1235         struct cfs_trace_page *tage;
1236         int i, cpu;
1237
1238         for_each_possible_cpu(cpu) {
1239                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
1240                         if (!tcd->tcd_pages_factor)
1241                                 /* Not initialised */
1242                                 continue;
1243                         tcd->tcd_shutting_down = 1;
1244
1245                         while (!list_empty(&tcd->tcd_pages)) {
1246                                 tage = list_first_entry(&tcd->tcd_pages,
1247                                                         struct cfs_trace_page,
1248                                                         linkage);
1249                                 __LASSERT_TAGE_INVARIANT(tage);
1250
1251                                 list_del(&tage->linkage);
1252                                 cfs_tage_free(tage);
1253                         }
1254                         tcd->tcd_cur_pages = 0;
1255                 }
1256         }
1257 }
1258
1259 static void cfs_trace_cleanup(void)
1260 {
1261         struct page_collection pc;
1262         int i;
1263
1264         INIT_LIST_HEAD(&pc.pc_pages);
1265
1266         trace_cleanup_on_all_cpus();
1267
1268         for (i = 0; i < CFS_TCD_TYPE_CNT && cfs_trace_data[i]; i++) {
1269                 kfree(cfs_trace_data[i]);
1270                 cfs_trace_data[i] = NULL;
1271         }
1272 }
1273
1274 void cfs_tracefile_exit(void)
1275 {
1276         cfs_trace_stop_thread();
1277         cfs_trace_cleanup();
1278 }