libcfs/libcfs/tracefile.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2012, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * libcfs/libcfs/tracefile.c
  33  *
  34  * Author: Zach Brown <zab@clusterfs.com>
  35  * Author: Phil Schwan <phil@clusterfs.com>
  36  */
  37
  38
  39 #define DEBUG_SUBSYSTEM S_LNET
  40 #define LUSTRE_TRACEFILE_PRIVATE
  41 #include "tracefile.h"
  42
  43 #include <linux/ctype.h>
  44 #include <linux/fs.h>
  45 #include <linux/kthread.h>
  46 #include <linux/pagemap.h>
  47 #include <linux/uaccess.h>
  48 #include <libcfs/linux/linux-fs.h>
  49 #include <libcfs/libcfs.h>
  50
  51 /* XXX move things up to the top, comment */
  52 union cfs_trace_data_union (*cfs_trace_data[TCD_MAX_TYPES])[NR_CPUS] __cacheline_aligned;
  53
  54 char cfs_tracefile[TRACEFILE_NAME_SIZE];
  55 long long cfs_tracefile_size = CFS_TRACEFILE_SIZE;
  56 static struct tracefiled_ctl trace_tctl;
  57 static DEFINE_MUTEX(cfs_trace_thread_mutex);
  58 static int thread_running = 0;
  59
  60 static atomic_t cfs_tage_allocated = ATOMIC_INIT(0);
  61 static DECLARE_RWSEM(cfs_tracefile_sem);
  62
  63 static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
  64                                         struct cfs_trace_cpu_data *tcd);
  65
  66 static inline struct cfs_trace_page *
  67 cfs_tage_from_list(struct list_head *list)
  68 {
  69         return list_entry(list, struct cfs_trace_page, linkage);
  70 }
  71
  72 static struct cfs_trace_page *cfs_tage_alloc(gfp_t gfp)
  73 {
  74         struct page            *page;
  75         struct cfs_trace_page *tage;
  76
  77         /* My caller is trying to free memory */
  78         if (!in_interrupt() && memory_pressure_get())
  79                 return NULL;
  80
  81         /*
  82          * Don't spam console with allocation failures: they will be reported
  83          * by upper layer anyway.
  84          */
  85         gfp |= __GFP_NOWARN;
  86         page = alloc_page(gfp);
  87         if (page == NULL)
  88                 return NULL;
  89
  90         tage = kmalloc(sizeof(*tage), gfp);
  91         if (tage == NULL) {
  92                 __free_page(page);
  93                 return NULL;
  94         }
  95
  96         tage->page = page;
  97         atomic_inc(&cfs_tage_allocated);
  98         return tage;
  99 }
 100
 101 static void cfs_tage_free(struct cfs_trace_page *tage)
 102 {
 103         __LASSERT(tage != NULL);
 104         __LASSERT(tage->page != NULL);
 105
 106         __free_page(tage->page);
 107         kfree(tage);
 108         atomic_dec(&cfs_tage_allocated);
 109 }
 110
 111 static void cfs_tage_to_tail(struct cfs_trace_page *tage,
 112                              struct list_head *queue)
 113 {
 114         __LASSERT(tage != NULL);
 115         __LASSERT(queue != NULL);
 116
 117         list_move_tail(&tage->linkage, queue);
 118 }
 119
 120 int cfs_trace_refill_stock(struct cfs_trace_cpu_data *tcd, gfp_t gfp,
 121                            struct list_head *stock)
 122 {
 123         int i;
 124
 125         /*
 126          * XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
 127          * from here: this will lead to infinite recursion.
 128          */
 129
 130         for (i = 0; i + tcd->tcd_cur_stock_pages < TCD_STOCK_PAGES ; ++ i) {
 131                 struct cfs_trace_page *tage;
 132
 133                 tage = cfs_tage_alloc(gfp);
 134                 if (tage == NULL)
 135                         break;
 136                 list_add_tail(&tage->linkage, stock);
 137         }
 138         return i;
 139 }
 140
 141 /* return a page that has 'len' bytes left at the end */
 142 static struct cfs_trace_page *
 143 cfs_trace_get_tage_try(struct cfs_trace_cpu_data *tcd, unsigned long len)
 144 {
 145         struct cfs_trace_page *tage;
 146
 147         if (tcd->tcd_cur_pages > 0) {
 148                 __LASSERT(!list_empty(&tcd->tcd_pages));
 149                 tage = cfs_tage_from_list(tcd->tcd_pages.prev);
 150                 if (tage->used + len <= PAGE_SIZE)
 151                         return tage;
 152         }
 153
 154         if (tcd->tcd_cur_pages < tcd->tcd_max_pages) {
 155                 if (tcd->tcd_cur_stock_pages > 0) {
 156                         tage = cfs_tage_from_list(tcd->tcd_stock_pages.prev);
 157                         --tcd->tcd_cur_stock_pages;
 158                         list_del_init(&tage->linkage);
 159                 } else {
 160                         tage = cfs_tage_alloc(GFP_ATOMIC);
 161                         if (unlikely(tage == NULL)) {
 162                                 if ((!memory_pressure_get() ||
 163                                      in_interrupt()) && printk_ratelimit())
 164                                         pr_warn("Lustre: cannot allocate a tage (%ld)\n",
 165                                                 tcd->tcd_cur_pages);
 166                                 return NULL;
 167                         }
 168                 }
 169
 170                 tage->used = 0;
 171                 tage->cpu = smp_processor_id();
 172                 tage->type = tcd->tcd_type;
 173                 list_add_tail(&tage->linkage, &tcd->tcd_pages);
 174                 tcd->tcd_cur_pages++;
 175
 176                 if (tcd->tcd_cur_pages > 8 && thread_running) {
 177                         struct tracefiled_ctl *tctl = &trace_tctl;
 178                         /*
 179                          * wake up tracefiled to process some pages.
 180                          */
 181                         wake_up(&tctl->tctl_waitq);
 182                 }
 183                 return tage;
 184         }
 185         return NULL;
 186 }
 187
 188 static void cfs_tcd_shrink(struct cfs_trace_cpu_data *tcd)
 189 {
 190         int pgcount = tcd->tcd_cur_pages / 10;
 191         struct page_collection pc;
 192         struct cfs_trace_page *tage;
 193         struct cfs_trace_page *tmp;
 194
 195         /*
 196          * XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
 197          * from here: this will lead to infinite recursion.
 198          */
 199
 200         if (printk_ratelimit())
 201                 pr_warn("Lustre: debug daemon buffer overflowed; discarding 10%% of pages (%d of %ld)\n",
 202                         pgcount + 1, tcd->tcd_cur_pages);
 203
 204         INIT_LIST_HEAD(&pc.pc_pages);
 205
 206         list_for_each_entry_safe(tage, tmp, &tcd->tcd_pages, linkage) {
 207                 if (pgcount-- == 0)
 208                         break;
 209
 210                 list_move_tail(&tage->linkage, &pc.pc_pages);
 211                 tcd->tcd_cur_pages--;
 212         }
 213         put_pages_on_tcd_daemon_list(&pc, tcd);
 214 }
 215
 216 /* return a page that has 'len' bytes left at the end */
 217 static struct cfs_trace_page *cfs_trace_get_tage(struct cfs_trace_cpu_data *tcd,
 218                                                  unsigned long len)
 219 {
 220         struct cfs_trace_page *tage;
 221
 222         /*
 223          * XXX nikita: do NOT call portals_debug_msg() (CDEBUG/ENTRY/EXIT)
 224          * from here: this will lead to infinite recursion.
 225          */
 226
 227         if (len > PAGE_SIZE) {
 228                 pr_err("LustreError: cowardly refusing to write %lu bytes in a page\n",
 229                        len);
 230                 return NULL;
 231         }
 232
 233         tage = cfs_trace_get_tage_try(tcd, len);
 234         if (tage != NULL)
 235                 return tage;
 236         if (thread_running)
 237                 cfs_tcd_shrink(tcd);
 238         if (tcd->tcd_cur_pages > 0) {
 239                 tage = cfs_tage_from_list(tcd->tcd_pages.next);
 240                 tage->used = 0;
 241                 cfs_tage_to_tail(tage, &tcd->tcd_pages);
 242         }
 243         return tage;
 244 }
 245
 246 int libcfs_debug_msg(struct libcfs_debug_msg_data *msgdata,
 247                      const char *format, ...)
 248 {
 249         struct cfs_trace_cpu_data *tcd = NULL;
 250         struct ptldebug_header header = {0};
 251         struct cfs_trace_page *tage;
 252         /* string_buf is used only if tcd != NULL, and is always set then */
 253         char *string_buf = NULL;
 254         char *debug_buf;
 255         int known_size;
 256         int needed = 85; /* seeded with average message length */
 257         int max_nob;
 258         va_list ap;
 259         int retry;
 260         int mask = msgdata->msg_mask;
 261         char *file = (char *)msgdata->msg_file;
 262         struct cfs_debug_limit_state *cdls = msgdata->msg_cdls;
 263
 264         if (strchr(file, '/'))
 265                 file = strrchr(file, '/') + 1;
 266
 267         tcd = cfs_trace_get_tcd();
 268
 269         /* cfs_trace_get_tcd() grabs a lock, which disables preemption and
 270          * pins us to a particular CPU.  This avoids an smp_processor_id()
 271          * warning on Linux when debugging is enabled.
 272          */
 273         cfs_set_ptldebug_header(&header, msgdata, CDEBUG_STACK());
 274
 275         if (!tcd)                /* arch may not log in IRQ context */
 276                 goto console;
 277
 278         if (tcd->tcd_cur_pages == 0)
 279                 header.ph_flags |= PH_FLAG_FIRST_RECORD;
 280
 281         if (tcd->tcd_shutting_down) {
 282                 cfs_trace_put_tcd(tcd);
 283                 tcd = NULL;
 284                 goto console;
 285         }
 286
 287         known_size = strlen(file) + 1;
 288         if (msgdata->msg_fn)
 289                 known_size += strlen(msgdata->msg_fn) + 1;
 290
 291         if (libcfs_debug_binary)
 292                 known_size += sizeof(header);
 293
 294         /*
 295          * May perform an additional pass to update 'needed' and increase
 296          * tage buffer size to match vsnprintf reported size required
 297          * On the second pass (retry=1) use vscnprintf [which returns
 298          * number of bytes written not including the terminating nul]
 299          * to clarify `needed` is used as number of bytes written
 300          * for the remainder of this function
 301          */
 302         for (retry = 0; retry < 2; retry++) {
 303                 tage = cfs_trace_get_tage(tcd, needed + known_size + 1);
 304                 if (!tage) {
 305                         if (needed + known_size > PAGE_SIZE)
 306                                 mask |= D_ERROR;
 307
 308                         cfs_trace_put_tcd(tcd);
 309                         tcd = NULL;
 310                         goto console;
 311                 }
 312
 313                 string_buf = (char *)page_address(tage->page) +
 314                              tage->used + known_size;
 315
 316                 max_nob = PAGE_SIZE - tage->used - known_size;
 317                 if (max_nob <= 0) {
 318                         pr_emerg("LustreError: negative max_nob: %d\n",
 319                                  max_nob);
 320                         mask |= D_ERROR;
 321                         cfs_trace_put_tcd(tcd);
 322                         tcd = NULL;
 323                         goto console;
 324                 }
 325
 326                 va_start(ap, format);
 327                 if (retry)
 328                         needed = vscnprintf(string_buf, max_nob, format, ap);
 329                 else
 330                         needed = vsnprintf(string_buf, max_nob, format, ap);
 331                 va_end(ap);
 332
 333                 if (needed < max_nob) /* well. printing ok.. */
 334                         break;
 335         }
 336
 337         /* `needed` is actual bytes written to string_buf */
 338         if (*(string_buf + needed - 1) != '\n') {
 339                 pr_info("Lustre: format at %s:%d:%s doesn't end in newline\n",
 340                         file, msgdata->msg_line, msgdata->msg_fn);
 341         } else if (mask & D_TTY) {
 342                 /* TTY needs '\r\n' to move carriage to leftmost position */
 343                 if (needed < 2 || *(string_buf + needed - 2) != '\r')
 344                         pr_info("Lustre: format at %s:%d:%s doesn't end in '\\r\\n'\n",
 345                                 file, msgdata->msg_line, msgdata->msg_fn);
 346         }
 347
 348         header.ph_len = known_size + needed;
 349         debug_buf = (char *)page_address(tage->page) + tage->used;
 350
 351         if (libcfs_debug_binary) {
 352                 memcpy(debug_buf, &header, sizeof(header));
 353                 tage->used += sizeof(header);
 354                 debug_buf += sizeof(header);
 355         }
 356
 357         strlcpy(debug_buf, file, PAGE_SIZE - tage->used);
 358         tage->used += strlen(file) + 1;
 359         debug_buf += strlen(file) + 1;
 360
 361         if (msgdata->msg_fn) {
 362                 strlcpy(debug_buf, msgdata->msg_fn, PAGE_SIZE - tage->used);
 363                 tage->used += strlen(msgdata->msg_fn) + 1;
 364                 debug_buf += strlen(msgdata->msg_fn) + 1;
 365         }
 366
 367         __LASSERT(debug_buf == string_buf);
 368
 369         tage->used += needed;
 370         __LASSERT(tage->used <= PAGE_SIZE);
 371
 372 console:
 373         if ((mask & libcfs_printk) == 0) {
 374                 /* no console output requested */
 375                 if (tcd != NULL)
 376                         cfs_trace_put_tcd(tcd);
 377                 return 1;
 378         }
 379
 380         if (cdls != NULL) {
 381                 if (libcfs_console_ratelimit &&
 382                     cdls->cdls_next != 0 &&     /* not first time ever */
 383                     time_before(jiffies, cdls->cdls_next)) {
 384                         /* skipping a console message */
 385                         cdls->cdls_count++;
 386                         if (tcd != NULL)
 387                                 cfs_trace_put_tcd(tcd);
 388                         return 1;
 389                 }
 390
 391                 if (time_after(jiffies, cdls->cdls_next +
 392                                         libcfs_console_max_delay +
 393                                         cfs_time_seconds(10))) {
 394                         /* last timeout was a long time ago */
 395                         cdls->cdls_delay /= libcfs_console_backoff * 4;
 396                 } else {
 397                         cdls->cdls_delay *= libcfs_console_backoff;
 398                 }
 399
 400                 if (cdls->cdls_delay < libcfs_console_min_delay)
 401                         cdls->cdls_delay = libcfs_console_min_delay;
 402                 else if (cdls->cdls_delay > libcfs_console_max_delay)
 403                         cdls->cdls_delay = libcfs_console_max_delay;
 404
 405                 /* ensure cdls_next is never zero after it's been seen */
 406                 cdls->cdls_next = (jiffies + cdls->cdls_delay) | 1;
 407         }
 408
 409         if (tcd) {
 410                 cfs_print_to_console(&header, mask, string_buf, needed, file,
 411                                      msgdata->msg_fn);
 412                 cfs_trace_put_tcd(tcd);
 413         } else {
 414                 string_buf = cfs_trace_get_console_buffer();
 415
 416                 va_start(ap, format);
 417                 needed = vscnprintf(string_buf, CFS_TRACE_CONSOLE_BUFFER_SIZE,
 418                                     format, ap);
 419                 va_end(ap);
 420
 421                 cfs_print_to_console(&header, mask,
 422                                      string_buf, needed, file, msgdata->msg_fn);
 423
 424                 put_cpu();
 425         }
 426
 427         if (cdls != NULL && cdls->cdls_count != 0) {
 428                 string_buf = cfs_trace_get_console_buffer();
 429
 430                 needed = scnprintf(string_buf, CFS_TRACE_CONSOLE_BUFFER_SIZE,
 431                                    "Skipped %d previous similar message%s\n",
 432                                    cdls->cdls_count,
 433                                    (cdls->cdls_count > 1) ? "s" : "");
 434
 435                 /* Do not allow print this to TTY */
 436                 cfs_print_to_console(&header, mask & ~D_TTY, string_buf,
 437                                      needed, file, msgdata->msg_fn);
 438
 439                 put_cpu();
 440                 cdls->cdls_count = 0;
 441         }
 442
 443         return 0;
 444 }
 445 EXPORT_SYMBOL(libcfs_debug_msg);
 446
 447 void
 448 cfs_trace_assertion_failed(const char *str,
 449                            struct libcfs_debug_msg_data *msgdata)
 450 {
 451         struct ptldebug_header hdr;
 452
 453         libcfs_panic_in_progress = 1;
 454         libcfs_catastrophe = 1;
 455         smp_mb();
 456
 457         cfs_set_ptldebug_header(&hdr, msgdata, CDEBUG_STACK());
 458
 459         cfs_print_to_console(&hdr, D_EMERG, str, strlen(str),
 460                              msgdata->msg_file, msgdata->msg_fn);
 461
 462         panic("Lustre debug assertion failure\n");
 463
 464         /* not reached */
 465 }
 466
 467 static void
 468 panic_collect_pages(struct page_collection *pc)
 469 {
 470         /* Do the collect_pages job on a single CPU: assumes that all other
 471          * CPUs have been stopped during a panic.  If this isn't true for some
 472          * arch, this will have to be implemented separately in each arch.  */
 473         int                        i;
 474         int                        j;
 475         struct cfs_trace_cpu_data *tcd;
 476
 477         INIT_LIST_HEAD(&pc->pc_pages);
 478
 479         cfs_tcd_for_each(tcd, i, j) {
 480                 list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
 481                 tcd->tcd_cur_pages = 0;
 482
 483                 if (pc->pc_want_daemon_pages) {
 484                         list_splice_init(&tcd->tcd_daemon_pages,
 485                                                 &pc->pc_pages);
 486                         tcd->tcd_cur_daemon_pages = 0;
 487                 }
 488         }
 489 }
 490
 491 static void collect_pages_on_all_cpus(struct page_collection *pc)
 492 {
 493         struct cfs_trace_cpu_data *tcd;
 494         int i, cpu;
 495
 496         for_each_possible_cpu(cpu) {
 497                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
 498                         list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
 499                         tcd->tcd_cur_pages = 0;
 500                         if (pc->pc_want_daemon_pages) {
 501                                 list_splice_init(&tcd->tcd_daemon_pages,
 502                                                         &pc->pc_pages);
 503                                 tcd->tcd_cur_daemon_pages = 0;
 504                         }
 505                 }
 506         }
 507 }
 508
 509 static void collect_pages(struct page_collection *pc)
 510 {
 511         INIT_LIST_HEAD(&pc->pc_pages);
 512
 513         if (libcfs_panic_in_progress)
 514                 panic_collect_pages(pc);
 515         else
 516                 collect_pages_on_all_cpus(pc);
 517 }
 518
 519 static void put_pages_back_on_all_cpus(struct page_collection *pc)
 520 {
 521         struct cfs_trace_cpu_data *tcd;
 522         struct list_head *cur_head;
 523         struct cfs_trace_page *tage;
 524         struct cfs_trace_page *tmp;
 525         int i, cpu;
 526
 527         for_each_possible_cpu(cpu) {
 528                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
 529                         cur_head = tcd->tcd_pages.next;
 530
 531                         list_for_each_entry_safe(tage, tmp, &pc->pc_pages,
 532                                                  linkage) {
 533
 534                                 __LASSERT_TAGE_INVARIANT(tage);
 535
 536                                 if (tage->cpu != cpu || tage->type != i)
 537                                         continue;
 538
 539                                 cfs_tage_to_tail(tage, cur_head);
 540                                 tcd->tcd_cur_pages++;
 541                         }
 542                 }
 543         }
 544 }
 545
 546 static void put_pages_back(struct page_collection *pc)
 547 {
 548         if (!libcfs_panic_in_progress)
 549                 put_pages_back_on_all_cpus(pc);
 550 }
 551
 552 /* Add pages to a per-cpu debug daemon ringbuffer.  This buffer makes sure that
 553  * we have a good amount of data at all times for dumping during an LBUG, even
 554  * if we have been steadily writing (and otherwise discarding) pages via the
 555  * debug daemon. */
 556 static void put_pages_on_tcd_daemon_list(struct page_collection *pc,
 557                                          struct cfs_trace_cpu_data *tcd)
 558 {
 559         struct cfs_trace_page *tage;
 560         struct cfs_trace_page *tmp;
 561
 562         list_for_each_entry_safe(tage, tmp, &pc->pc_pages, linkage) {
 563                 __LASSERT_TAGE_INVARIANT(tage);
 564
 565                 if (tage->cpu != tcd->tcd_cpu || tage->type != tcd->tcd_type)
 566                         continue;
 567
 568                 cfs_tage_to_tail(tage, &tcd->tcd_daemon_pages);
 569                 tcd->tcd_cur_daemon_pages++;
 570
 571                 if (tcd->tcd_cur_daemon_pages > tcd->tcd_max_pages) {
 572                         struct cfs_trace_page *victim;
 573
 574                         __LASSERT(!list_empty(&tcd->tcd_daemon_pages));
 575                         victim = cfs_tage_from_list(tcd->tcd_daemon_pages.next);
 576
 577                         __LASSERT_TAGE_INVARIANT(victim);
 578
 579                         list_del(&victim->linkage);
 580                         cfs_tage_free(victim);
 581                         tcd->tcd_cur_daemon_pages--;
 582                 }
 583         }
 584 }
 585
 586 static void put_pages_on_daemon_list(struct page_collection *pc)
 587 {
 588         struct cfs_trace_cpu_data *tcd;
 589         int i, cpu;
 590
 591         for_each_possible_cpu(cpu) {
 592                 cfs_tcd_for_each_type_lock(tcd, i, cpu)
 593                         put_pages_on_tcd_daemon_list(pc, tcd);
 594         }
 595 }
 596
 597 void cfs_trace_debug_print(void)
 598 {
 599         struct page_collection pc;
 600         struct cfs_trace_page *tage;
 601         struct cfs_trace_page *tmp;
 602
 603         pc.pc_want_daemon_pages = 1;
 604         collect_pages(&pc);
 605         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 606                 char *p, *file, *fn;
 607                 struct page *page;
 608
 609                 __LASSERT_TAGE_INVARIANT(tage);
 610
 611                 page = tage->page;
 612                 p = page_address(page);
 613                 while (p < ((char *)page_address(page) + tage->used)) {
 614                         struct ptldebug_header *hdr;
 615                         int len;
 616                         hdr = (void *)p;
 617                         p += sizeof(*hdr);
 618                         file = p;
 619                         p += strlen(file) + 1;
 620                         fn = p;
 621                         p += strlen(fn) + 1;
 622                         len = hdr->ph_len - (int)(p - (char *)hdr);
 623
 624                         cfs_print_to_console(hdr, D_EMERG, p, len, file, fn);
 625
 626                         p += len;
 627                 }
 628
 629                 list_del(&tage->linkage);
 630                 cfs_tage_free(tage);
 631         }
 632 }
 633
 634 int cfs_tracefile_dump_all_pages(char *filename)
 635 {
 636         struct page_collection  pc;
 637         struct file             *filp;
 638         struct cfs_trace_page   *tage;
 639         struct cfs_trace_page   *tmp;
 640         char                    *buf;
 641         int rc;
 642
 643         down_write(&cfs_tracefile_sem);
 644
 645         filp = filp_open(filename, O_CREAT|O_EXCL|O_WRONLY|O_LARGEFILE, 0600);
 646         if (IS_ERR(filp)) {
 647                 rc = PTR_ERR(filp);
 648                 filp = NULL;
 649                 pr_err("LustreError: can't open %s for dump: rc = %d\n",
 650                       filename, rc);
 651                 goto out;
 652         }
 653
 654         pc.pc_want_daemon_pages = 1;
 655         collect_pages(&pc);
 656         if (list_empty(&pc.pc_pages)) {
 657                 rc = 0;
 658                 goto close;
 659         }
 660
 661         /* ok, for now, just write the pages.  in the future we'll be building
 662          * iobufs with the pages and calling generic_direct_IO */
 663         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 664
 665                 __LASSERT_TAGE_INVARIANT(tage);
 666
 667                 buf = kmap(tage->page);
 668                 rc = cfs_kernel_write(filp, buf, tage->used, &filp->f_pos);
 669                 kunmap(tage->page);
 670                 if (rc != (int)tage->used) {
 671                         pr_warn("Lustre: wanted to write %u but wrote %d\n",
 672                                 tage->used, rc);
 673                         put_pages_back(&pc);
 674                         __LASSERT(list_empty(&pc.pc_pages));
 675                         break;
 676                 }
 677                 list_del(&tage->linkage);
 678                 cfs_tage_free(tage);
 679         }
 680
 681         rc = vfs_fsync_range(filp, 0, LLONG_MAX, 1);
 682         if (rc)
 683                 pr_err("LustreError: sync returns: rc = %d\n", rc);
 684 close:
 685         filp_close(filp, NULL);
 686 out:
 687         up_write(&cfs_tracefile_sem);
 688         return rc;
 689 }
 690
 691 void cfs_trace_flush_pages(void)
 692 {
 693         struct page_collection pc;
 694         struct cfs_trace_page *tage;
 695         struct cfs_trace_page *tmp;
 696
 697         pc.pc_want_daemon_pages = 1;
 698         collect_pages(&pc);
 699         list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 700
 701                 __LASSERT_TAGE_INVARIANT(tage);
 702
 703                 list_del(&tage->linkage);
 704                 cfs_tage_free(tage);
 705         }
 706 }
 707
 708 int cfs_trace_copyin_string(char *knl_buffer, int knl_buffer_nob,
 709                             const char __user *usr_buffer, int usr_buffer_nob)
 710 {
 711         int    nob;
 712
 713         if (usr_buffer_nob > knl_buffer_nob)
 714                 return -EOVERFLOW;
 715
 716         if (copy_from_user(knl_buffer, usr_buffer, usr_buffer_nob))
 717                 return -EFAULT;
 718
 719         nob = strnlen(knl_buffer, usr_buffer_nob);
 720         while (--nob >= 0)                      /* strip trailing whitespace */
 721                 if (!isspace(knl_buffer[nob]))
 722                         break;
 723
 724         if (nob < 0)                            /* empty string */
 725                 return -EINVAL;
 726
 727         if (nob == knl_buffer_nob)              /* no space to terminate */
 728                 return -EOVERFLOW;
 729
 730         knl_buffer[nob + 1] = 0;                /* terminate */
 731         return 0;
 732 }
 733 EXPORT_SYMBOL(cfs_trace_copyin_string);
 734
 735 int cfs_trace_copyout_string(char __user *usr_buffer, int usr_buffer_nob,
 736                              const char *knl_buffer, char *append)
 737 {
 738         /* NB if 'append' != NULL, it's a single character to append to the
 739          * copied out string - usually "\n", for /proc entries and "" (i.e. a
 740          * terminating zero byte) for sysctl entries */
 741         int   nob = strlen(knl_buffer);
 742
 743         if (nob > usr_buffer_nob)
 744                 nob = usr_buffer_nob;
 745
 746         if (copy_to_user(usr_buffer, knl_buffer, nob))
 747                 return -EFAULT;
 748
 749         if (append != NULL && nob < usr_buffer_nob) {
 750                 if (copy_to_user(usr_buffer + nob, append, 1))
 751                         return -EFAULT;
 752
 753                 nob++;
 754         }
 755
 756         return nob;
 757 }
 758 EXPORT_SYMBOL(cfs_trace_copyout_string);
 759
 760 int cfs_trace_allocate_string_buffer(char **str, int nob)
 761 {
 762         if (nob > 2 * PAGE_SIZE)        /* string must be "sensible" */
 763                 return -EINVAL;
 764
 765         *str = kmalloc(nob, GFP_KERNEL | __GFP_ZERO);
 766         if (*str == NULL)
 767                 return -ENOMEM;
 768
 769         return 0;
 770 }
 771
 772 int cfs_trace_dump_debug_buffer_usrstr(void __user *usr_str, int usr_str_nob)
 773 {
 774         char         *str;
 775         int           rc;
 776
 777         rc = cfs_trace_allocate_string_buffer(&str, usr_str_nob + 1);
 778         if (rc != 0)
 779                 return rc;
 780
 781         rc = cfs_trace_copyin_string(str, usr_str_nob + 1,
 782                                      usr_str, usr_str_nob);
 783         if (rc != 0)
 784                 goto out;
 785
 786         if (str[0] != '/') {
 787                 rc = -EINVAL;
 788                 goto out;
 789         }
 790         rc = cfs_tracefile_dump_all_pages(str);
 791 out:
 792         kfree(str);
 793         return rc;
 794 }
 795
 796 int cfs_trace_daemon_command(char *str)
 797 {
 798         int       rc = 0;
 799
 800         down_write(&cfs_tracefile_sem);
 801
 802         if (strcmp(str, "stop") == 0) {
 803                 up_write(&cfs_tracefile_sem);
 804                 cfs_trace_stop_thread();
 805                 down_write(&cfs_tracefile_sem);
 806                 memset(cfs_tracefile, 0, sizeof(cfs_tracefile));
 807
 808         } else if (strncmp(str, "size=", 5) == 0) {
 809                 unsigned long tmp;
 810
 811                 rc = kstrtoul(str + 5, 10, &tmp);
 812                 if (!rc) {
 813                         if (tmp < 10 || tmp > 20480)
 814                                 cfs_tracefile_size = CFS_TRACEFILE_SIZE;
 815                         else
 816                                 cfs_tracefile_size = tmp << 20;
 817                 }
 818         } else if (strlen(str) >= sizeof(cfs_tracefile)) {
 819                 rc = -ENAMETOOLONG;
 820         } else if (str[0] != '/') {
 821                 rc = -EINVAL;
 822         } else {
 823                 strcpy(cfs_tracefile, str);
 824
 825                 pr_info("Lustre: debug daemon will attempt to start writing to %s (%lukB max)\n",
 826                         cfs_tracefile, (long)(cfs_tracefile_size >> 10));
 827
 828                 cfs_trace_start_thread();
 829         }
 830
 831         up_write(&cfs_tracefile_sem);
 832         return rc;
 833 }
 834
 835 int cfs_trace_daemon_command_usrstr(void __user *usr_str, int usr_str_nob)
 836 {
 837         char *str;
 838         int   rc;
 839
 840         rc = cfs_trace_allocate_string_buffer(&str, usr_str_nob + 1);
 841         if (rc != 0)
 842                 return rc;
 843
 844         rc = cfs_trace_copyin_string(str, usr_str_nob + 1,
 845                                  usr_str, usr_str_nob);
 846         if (rc == 0)
 847                 rc = cfs_trace_daemon_command(str);
 848
 849         kfree(str);
 850         return rc;
 851 }
 852
 853 int cfs_trace_set_debug_mb(int mb)
 854 {
 855         int i;
 856         int j;
 857         unsigned long pages;
 858         unsigned long total_mb = (cfs_totalram_pages() >> (20 - PAGE_SHIFT));
 859         unsigned long limit = max_t(unsigned long, 512, (total_mb * 4) / 5);
 860         struct cfs_trace_cpu_data *tcd;
 861
 862         if (mb < num_possible_cpus()) {
 863                 pr_warn("Lustre: %d MB is too small for debug buffer size, setting it to %d MB.\n",
 864                         mb, num_possible_cpus());
 865                 mb = num_possible_cpus();
 866         }
 867
 868         if (mb > limit) {
 869                 pr_warn("Lustre: %d MB is too large for debug buffer size, setting it to %lu MB.\n",
 870                         mb, limit);
 871                 mb = limit;
 872         }
 873
 874         mb /= num_possible_cpus();
 875         pages = mb << (20 - PAGE_SHIFT);
 876
 877         down_write(&cfs_tracefile_sem);
 878
 879         cfs_tcd_for_each(tcd, i, j)
 880                 tcd->tcd_max_pages = (pages * tcd->tcd_pages_factor) / 100;
 881
 882         up_write(&cfs_tracefile_sem);
 883
 884         return mb;
 885 }
 886
 887 int cfs_trace_get_debug_mb(void)
 888 {
 889         int i;
 890         int j;
 891         struct cfs_trace_cpu_data *tcd;
 892         int total_pages = 0;
 893
 894         down_read(&cfs_tracefile_sem);
 895
 896         cfs_tcd_for_each(tcd, i, j)
 897                 total_pages += tcd->tcd_max_pages;
 898
 899         up_read(&cfs_tracefile_sem);
 900
 901         return (total_pages >> (20 - PAGE_SHIFT)) + 1;
 902 }
 903
 904 static int tracefiled(void *arg)
 905 {
 906         struct page_collection pc;
 907         struct tracefiled_ctl *tctl = arg;
 908         struct cfs_trace_page *tage;
 909         struct cfs_trace_page *tmp;
 910         struct file *filp;
 911         char *buf;
 912         int last_loop = 0;
 913         int rc;
 914
 915         /* we're started late enough that we pick up init's fs context */
 916         /* this is so broken in uml?  what on earth is going on? */
 917
 918         complete(&tctl->tctl_start);
 919
 920         while (1) {
 921                 wait_queue_entry_t __wait;
 922
 923                 pc.pc_want_daemon_pages = 0;
 924                 collect_pages(&pc);
 925                 if (list_empty(&pc.pc_pages))
 926                         goto end_loop;
 927
 928                 filp = NULL;
 929                 down_read(&cfs_tracefile_sem);
 930                 if (cfs_tracefile[0] != 0) {
 931                         filp = filp_open(cfs_tracefile,
 932                                          O_CREAT | O_RDWR | O_LARGEFILE,
 933                                          0600);
 934                         if (IS_ERR(filp)) {
 935                                 rc = PTR_ERR(filp);
 936                                 filp = NULL;
 937                                 pr_warn("Lustre: couldn't open %s: rc = %d\n",
 938                                         cfs_tracefile, rc);
 939                         }
 940                 }
 941                 up_read(&cfs_tracefile_sem);
 942                 if (filp == NULL) {
 943                         put_pages_on_daemon_list(&pc);
 944                         __LASSERT(list_empty(&pc.pc_pages));
 945                         goto end_loop;
 946                 }
 947
 948                 list_for_each_entry_safe(tage, tmp, &pc.pc_pages, linkage) {
 949                         struct dentry *de = file_dentry(filp);
 950                         static loff_t f_pos;
 951
 952                         __LASSERT_TAGE_INVARIANT(tage);
 953
 954                         if (f_pos >= (off_t)cfs_tracefile_size)
 955                                 f_pos = 0;
 956                         else if (f_pos > i_size_read(de->d_inode))
 957                                 f_pos = i_size_read(de->d_inode);
 958
 959                         buf = kmap(tage->page);
 960                         rc = cfs_kernel_write(filp, buf, tage->used, &f_pos);
 961                         kunmap(tage->page);
 962                         if (rc != (int)tage->used) {
 963                                 pr_warn("Lustre: wanted to write %u but wrote %d\n",
 964                                         tage->used, rc);
 965                                 put_pages_back(&pc);
 966                                 __LASSERT(list_empty(&pc.pc_pages));
 967                                 break;
 968                         }
 969                 }
 970
 971                 filp_close(filp, NULL);
 972                 put_pages_on_daemon_list(&pc);
 973                 if (!list_empty(&pc.pc_pages)) {
 974                         int i;
 975
 976                         pr_alert("Lustre: trace pages aren't empty\n");
 977                         pr_err("Lustre: total cpus(%d): ", num_possible_cpus());
 978                         for (i = 0; i < num_possible_cpus(); i++)
 979                                 if (cpu_online(i))
 980                                         pr_cont("%d(on) ", i);
 981                                 else
 982                                         pr_cont("%d(off) ", i);
 983                         pr_cont("\n");
 984
 985                         i = 0;
 986                         list_for_each_entry_safe(tage, tmp, &pc.pc_pages,
 987                                                  linkage)
 988                                 pr_err("Lustre: page %d belongs to cpu %d\n",
 989                                        ++i, tage->cpu);
 990                         pr_err("Lustre: There are %d pages unwritten\n", i);
 991                 }
 992                 __LASSERT(list_empty(&pc.pc_pages));
 993 end_loop:
 994                 if (atomic_read(&tctl->tctl_shutdown)) {
 995                         if (last_loop == 0) {
 996                                 last_loop = 1;
 997                                 continue;
 998                         } else {
 999                                 break;
1000                         }
1001                 }
1002                 init_waitqueue_entry(&__wait, current);
1003                 add_wait_queue(&tctl->tctl_waitq, &__wait);
1004                 schedule_timeout_interruptible(cfs_time_seconds(1));
1005                 remove_wait_queue(&tctl->tctl_waitq, &__wait);
1006         }
1007         complete(&tctl->tctl_stop);
1008         return 0;
1009 }
1010
1011 int cfs_trace_start_thread(void)
1012 {
1013         struct tracefiled_ctl *tctl = &trace_tctl;
1014         int rc = 0;
1015
1016         mutex_lock(&cfs_trace_thread_mutex);
1017         if (thread_running)
1018                 goto out;
1019
1020         init_completion(&tctl->tctl_start);
1021         init_completion(&tctl->tctl_stop);
1022         init_waitqueue_head(&tctl->tctl_waitq);
1023         atomic_set(&tctl->tctl_shutdown, 0);
1024
1025         if (IS_ERR(kthread_run(tracefiled, tctl, "ktracefiled"))) {
1026                 rc = -ECHILD;
1027                 goto out;
1028         }
1029
1030         wait_for_completion(&tctl->tctl_start);
1031         thread_running = 1;
1032 out:
1033         mutex_unlock(&cfs_trace_thread_mutex);
1034         return rc;
1035 }
1036
1037 void cfs_trace_stop_thread(void)
1038 {
1039         struct tracefiled_ctl *tctl = &trace_tctl;
1040
1041         mutex_lock(&cfs_trace_thread_mutex);
1042         if (thread_running) {
1043                 pr_info("Lustre: shutting down debug daemon thread...\n");
1044                 atomic_set(&tctl->tctl_shutdown, 1);
1045                 wait_for_completion(&tctl->tctl_stop);
1046                 thread_running = 0;
1047         }
1048         mutex_unlock(&cfs_trace_thread_mutex);
1049 }
1050
1051 int cfs_tracefile_init(int max_pages)
1052 {
1053         struct cfs_trace_cpu_data *tcd;
1054         int     i;
1055         int     j;
1056         int     rc;
1057         int     factor;
1058
1059         rc = cfs_tracefile_init_arch();
1060         if (rc != 0)
1061                 return rc;
1062
1063         cfs_tcd_for_each(tcd, i, j) {
1064                 /* tcd_pages_factor is initialized int tracefile_init_arch. */
1065                 factor = tcd->tcd_pages_factor;
1066                 INIT_LIST_HEAD(&tcd->tcd_pages);
1067                 INIT_LIST_HEAD(&tcd->tcd_stock_pages);
1068                 INIT_LIST_HEAD(&tcd->tcd_daemon_pages);
1069                 tcd->tcd_cur_pages = 0;
1070                 tcd->tcd_cur_stock_pages = 0;
1071                 tcd->tcd_cur_daemon_pages = 0;
1072                 tcd->tcd_max_pages = (max_pages * factor) / 100;
1073                 LASSERT(tcd->tcd_max_pages > 0);
1074                 tcd->tcd_shutting_down = 0;
1075         }
1076         return 0;
1077 }
1078
1079 static void trace_cleanup_on_all_cpus(void)
1080 {
1081         struct cfs_trace_cpu_data *tcd;
1082         struct cfs_trace_page *tage;
1083         struct cfs_trace_page *tmp;
1084         int i, cpu;
1085
1086         for_each_possible_cpu(cpu) {
1087                 cfs_tcd_for_each_type_lock(tcd, i, cpu) {
1088                         tcd->tcd_shutting_down = 1;
1089
1090                         list_for_each_entry_safe(tage, tmp, &tcd->tcd_pages, linkage) {
1091                                 __LASSERT_TAGE_INVARIANT(tage);
1092
1093                                 list_del(&tage->linkage);
1094                                 cfs_tage_free(tage);
1095                         }
1096                         tcd->tcd_cur_pages = 0;
1097                 }
1098         }
1099 }
1100
1101 static void cfs_trace_cleanup(void)
1102 {
1103         struct page_collection pc;
1104
1105         INIT_LIST_HEAD(&pc.pc_pages);
1106
1107         trace_cleanup_on_all_cpus();
1108
1109         cfs_tracefile_fini_arch();
1110 }
1111
1112 void cfs_tracefile_exit(void)
1113 {
1114         cfs_trace_stop_thread();
1115         cfs_trace_cleanup();
1116 }