4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.gnu.org/licenses/gpl-2.0.html
22 * Copyright 2020, DataDirect Networks Storage.
24 * This file is part of Lustre, http://www.lustre.org/
26 * Author: John L. Hammond <jhammond@whamcloud.com>
28 * lustre/utils/ofd_access_log_reader.c
30 * Sample utility to discover and read Lustre (ofd) access logs.
32 * This demonstrates the discovery and reading of Lustre access logs
33 * (see linux/lustre/lustre_access_log.h and
34 * lustre/ofd/ofd_access_log.c.). By default it opens the control
35 * device, discovers and opens all access log devices, and consumes
36 * all access log entries. If invoked with the --list option then it
37 * prints information about all available devices to stdout and exits.
39 * Structured trace points (when --trace is used) are added to permit
40 * testing of the access log functionality (see test_165* in
41 * lustre/tests/sanity.sh).
57 #include <sys/epoll.h>
58 #include <sys/ioctl.h>
59 #include <sys/signalfd.h>
61 #include <sys/sysmacros.h>
62 #include <sys/types.h>
63 #include <linux/types.h>
64 #include <linux/lustre/lustre_user.h>
65 #include <linux/lustre/lustre_access_log.h>
68 /* TODO fsname filter */
70 static FILE *debug_file;
71 static FILE *trace_file;
73 #define DEBUG(fmt, args...) \
75 if (debug_file != NULL) \
76 fprintf(debug_file, "DEBUG %s:%d: "fmt, __func__, __LINE__, ##args); \
79 #define TRACE(fmt, args...) \
81 if (trace_file != NULL) \
82 fprintf(trace_file, "TRACE "fmt, ##args); \
85 #define DEBUG_D(x) DEBUG("%s = %d\n", #x, x)
86 #define DEBUG_P(x) DEBUG("%s = %p\n", #x, x)
87 #define DEBUG_S(x) DEBUG("%s = '%s'\n", #x, x)
88 #define DEBUG_U(x) DEBUG("%s = %u\n", #x, x)
89 #define DEBUG_U32(x) DEBUG("%s = %"PRIu32"\n", #x, x)
90 #define DEBUG_U64(x) DEBUG("%s = %"PRIu64"\n", #x, x)
92 #define ERROR(fmt, args...) \
93 fprintf(stderr, "%s: "fmt, program_invocation_short_name, ##args)
95 #define FATAL(fmt, args...) \
97 ERROR("FATAL: "fmt, ##args); \
102 ALR_EXIT_SUCCESS = INT_MIN + EXIT_SUCCESS,
103 ALR_EXIT_FAILURE = INT_MIN + EXIT_FAILURE,
111 int (*alr_io)(int /* epoll_fd */, struct alr_dev * /* this */, unsigned int /* mask */);
112 void (*alr_destroy)(struct alr_dev *);
117 struct alr_dev alr_dev;
120 size_t alr_entry_size;
124 static struct alr_log *alr_log[1 << 20]; /* 20 == MINORBITS */
125 static int oal_version; /* FIXME ... major version, minor version */
126 static unsigned int oal_log_major;
127 static unsigned int oal_log_minor_max;
129 #define D_ALR_DEV "%s %d"
130 #define P_ALR_DEV(ad) \
131 (ad)->alr_name, (ad)->alr_fd
133 #define D_ALR_LOG D_ALR_DEV" %u:%u"
134 #define P_ALR_LOG(al) \
135 P_ALR_DEV(&(al)->alr_dev), major((al)->alr_rdev), minor((al)->alr_rdev)
137 static void alr_dev_free(int epoll_fd, struct alr_dev *ad)
139 TRACE("alr_dev_free %s\n", ad->alr_name);
141 if (!(ad->alr_fd < 0))
142 epoll_ctl(epoll_fd, EPOLL_CTL_DEL, ad->alr_fd, NULL);
144 if (ad->alr_destroy != NULL)
145 (*ad->alr_destroy)(ad);
147 if (!(ad->alr_fd < 0))
154 static struct alr_log **alr_log_lookup(dev_t rdev)
156 assert(major(rdev) == oal_log_major);
158 if (!(minor(rdev) < ARRAY_SIZE(alr_log)))
161 return &alr_log[minor(rdev)];
164 static const char *alr_flags_to_str(unsigned int flags)
166 switch (flags & (OFD_ACCESS_READ | OFD_ACCESS_WRITE)) {
169 case OFD_ACCESS_READ:
171 case OFD_ACCESS_WRITE:
173 case OFD_ACCESS_READ | OFD_ACCESS_WRITE:
178 /* /dev/lustre-access-log/scratch-OST0000 device poll callback: read entries
179 * from log and print. */
180 static int alr_log_io(int epoll_fd, struct alr_dev *ad, unsigned int mask)
182 struct alr_log *al = container_of(ad, struct alr_log, alr_dev);
185 TRACE("alr_log_io %s\n", ad->alr_name);
188 assert(al->alr_entry_size != 0);
189 assert(al->alr_buf_size != 0);
190 assert(al->alr_buf != NULL);
192 count = read(ad->alr_fd, al->alr_buf, al->alr_buf_size);
194 ERROR("cannot read events from '%s': %s\n", ad->alr_name, strerror(errno));
199 TRACE("alr_log_eof %s\n", ad->alr_name);
203 if (count % al->alr_entry_size != 0) {
204 ERROR("invalid read from "D_ALR_LOG": entry_size = %zu, count = %zd\n",
205 P_ALR_LOG(al), al->alr_entry_size, count);
209 DEBUG("read "D_ALR_LOG", count = %zd\n", P_ALR_LOG(al), count);
211 for (i = 0; i < count; i += al->alr_entry_size) {
212 struct ofd_access_entry_v1 *oae =
213 (struct ofd_access_entry_v1 *)&al->alr_buf[i];
215 TRACE("alr_log_entry %s "DFID" %lu %lu %lu %u %u %s\n",
217 PFID(&oae->oae_parent_fid),
218 (unsigned long)oae->oae_begin,
219 (unsigned long)oae->oae_end,
220 (unsigned long)oae->oae_time,
221 (unsigned int)oae->oae_size,
222 (unsigned int)oae->oae_segment_count,
223 alr_flags_to_str(oae->oae_flags));
229 static void alr_log_destroy(struct alr_dev *ad)
231 struct alr_log *al = container_of(ad, struct alr_log, alr_dev);
232 struct alr_log **pal;
234 TRACE("alr_log_free %s\n", ad->alr_name);
235 assert(major(al->alr_rdev) == oal_log_major);
237 pal = alr_log_lookup(al->alr_rdev);
238 if (pal != NULL && *pal == al)
243 al->alr_buf_size = 0;
246 /* Add an access log (identified by path) to the epoll set. */
247 static int alr_log_add(int epoll_fd, const char *path)
249 struct alr_log **pal, *al = NULL;
254 fd = open(path, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
256 ERROR("cannot open device '%s': %s\n", path, strerror(errno));
257 rc = (errno == ENOENT ? 0 : -1); /* Possible race. */
261 /* Revalidate rdev in case of race. */
264 ERROR("cannot stat '%s': %s\n", path, strerror(errno));
268 if (major(st.st_rdev) != oal_log_major)
271 pal = alr_log_lookup(st.st_rdev);
273 ERROR("no device slot available for '%s' with minor %u\n",
274 path, minor(st.st_rdev));
279 goto out; /* We already have this device. */
281 struct lustre_access_log_info_v1 lali;
283 memset(&lali, 0, sizeof(lali));
285 rc = ioctl(fd, LUSTRE_ACCESS_LOG_IOCTL_INFO, &lali);
287 ERROR("cannot get info for device '%s': %s\n",
288 path, strerror(errno));
292 if (lali.lali_type != LUSTRE_ACCESS_LOG_TYPE_OFD) {
297 al = calloc(1, sizeof(*al));
299 FATAL("cannot allocate struct alr_dev of size %zu: %s\n",
300 sizeof(*al), strerror(errno));
302 al->alr_dev.alr_io = &alr_log_io;
303 al->alr_dev.alr_destroy = &alr_log_destroy;
304 al->alr_dev.alr_fd = fd;
307 al->alr_rdev = st.st_rdev;
309 al->alr_dev.alr_name = strdup(lali.lali_name);
310 if (al->alr_dev.alr_name == NULL)
311 FATAL("cannot copy name of size %zu: %s\n",
312 strlen(lali.lali_name), strerror(errno));
314 al->alr_buf_size = lali.lali_log_size;
315 al->alr_entry_size = lali.lali_entry_size;
317 if (al->alr_entry_size == 0) {
318 ERROR("device '%s' has zero entry size\n", path);
323 if (al->alr_buf_size == 0)
324 al->alr_buf_size = 1048576;
326 al->alr_buf_size = roundup(al->alr_buf_size, al->alr_entry_size);
328 al->alr_buf = malloc(al->alr_buf_size);
329 if (al->alr_buf == NULL)
330 FATAL("cannot allocate log buffer for '%s' of size %zu: %s\n",
331 path, al->alr_buf_size, strerror(errno));
333 struct epoll_event ev = {
334 .events = EPOLLIN | EPOLLHUP,
335 .data.ptr = &al->alr_dev,
338 rc = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, al->alr_dev.alr_fd, &ev);
340 ERROR("cannot add device '%s' to epoll set: %s\n",
341 path, strerror(errno));
345 TRACE("alr_log_add %s\n", al->alr_dev.alr_name);
347 if (oal_log_minor_max < minor(al->alr_rdev))
348 oal_log_minor_max = minor(al->alr_rdev);
350 assert(*pal == NULL);
356 alr_dev_free(epoll_fd, &al->alr_dev);
364 /* Scan /dev/lustre-access-log/ for new access log devices and add to
366 static int alr_scan(int epoll_fd)
368 const char dir_path[] = "/dev/"LUSTRE_ACCESS_LOG_DIR_NAME;
374 dir = opendir(dir_path);
376 ERROR("cannot open '%s' for scanning: %s\n", dir_path, strerror(errno));
377 return ALR_EXIT_FAILURE;
382 /* Scan /dev for devices with major equal to oal_log_major and add
383 * any new devices. */
384 while ((d = readdir(dir)) != NULL) {
385 char path[6 + PATH_MAX];
386 struct alr_log **pal;
389 if (d->d_type != DT_CHR)
392 rc = fstatat(dir_fd, d->d_name, &st, 0);
394 ERROR("cannot stat '%s/%s' while scanning: %s\n",
395 dir_path, d->d_name, strerror(errno));
399 if (!S_ISCHR(st.st_mode))
402 if (major(st.st_rdev) != oal_log_major)
405 pal = alr_log_lookup(st.st_rdev);
407 ERROR("no device slot available for '%s/%s' with minor %u\n",
408 dir_path, d->d_name, minor(st.st_rdev));
413 continue; /* We already have this device. */
415 snprintf(path, sizeof(path), "%s/%s", dir_path, d->d_name);
417 alr_log_add(epoll_fd, path);
425 /* /dev/lustre-access-log/control device poll callback: call prescan
426 * ioctl and scan /dev/lustre-access-log/ for new access log
428 static int alr_ctl_io(int epoll_fd, struct alr_dev *cd, unsigned int mask)
432 TRACE("%s\n", __func__);
436 return ALR_EXIT_FAILURE;
439 return ALR_EXIT_SUCCESS;
441 rc = ioctl(cd->alr_fd, LUSTRE_ACCESS_LOG_IOCTL_PRESCAN);
443 ERROR("cannot start scanning: %s\n", strerror(errno));
444 return ALR_EXIT_FAILURE;
447 return alr_scan(epoll_fd);
450 /* signalfd epoll callback. Handle SIGINT and SIGTERM by breaking from
451 * the epoll loop and exiting normally.*/
452 static int alr_signal_io(int epoll_fd, struct alr_dev *sd, unsigned int mask)
454 struct signalfd_siginfo ssi;
457 TRACE("%s\n", __func__);
460 rc = read(sd->alr_fd, &ssi, sizeof(ssi));
464 DEBUG_U32(ssi.ssi_signo);
465 switch (ssi.ssi_signo) {
468 return ALR_EXIT_SUCCESS;
474 /* Call LUSTRE_ACCESS_LOG_IOCTL_INFO to get access log info and print
475 * YAML formatted info to stdout. */
476 static int alr_log_info(struct alr_log *al)
478 struct lustre_access_log_info_v1 lali;
481 rc = ioctl(al->alr_dev.alr_fd, LUSTRE_ACCESS_LOG_IOCTL_INFO, &lali);
483 ERROR("cannot get info for device '%s': %s\n",
484 al->alr_dev.alr_name, strerror(errno));
488 printf("- name: %s\n"
495 " _entry_space: %u\n"
496 " _entry_count: %u\n"
503 lali.lali_entry_size,
506 lali._lali_entry_space,
507 lali._lali_entry_count,
508 lali._lali_drop_count,
509 lali._lali_is_closed);
514 static struct alr_dev *alr_dev_create(int epoll_fd, int fd, const char *name,
515 int (*io)(int, struct alr_dev *, unsigned int),
516 void (*destroy)(struct alr_dev *))
521 alr = calloc(1, sizeof(*alr));
525 alr->alr_name = strdup(name);
526 if (alr->alr_name == NULL) {
531 alr->alr_destroy = destroy;
534 struct epoll_event event = {
535 .events = EPOLLIN | EPOLLHUP,
539 rc = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, alr->alr_fd, &event);
548 int main(int argc, char *argv[])
550 const char ctl_path[] = "/dev/"LUSTRE_ACCESS_LOG_DIR_NAME"/control";
551 struct alr_dev *alr_signal = NULL;
552 struct alr_dev *alr_ctl = NULL;
562 static struct option options[] = {
563 { .name = "debug", .has_arg = optional_argument, .val = 'd', },
564 { .name = "help", .has_arg = no_argument, .val = 'h', },
565 { .name = "list", .has_arg = no_argument, .val = 'l', },
566 { .name = "trace", .has_arg = optional_argument, .val = 't', },
570 while ((c = getopt_long(argc, argv, "d::hlt::", options, NULL)) != -1) {
573 if (optarg == NULL) {
575 } else if (strcmp(optarg, "-") == 0) {
578 debug_file = fopen(optarg, "a");
579 if (debug_file == NULL)
580 FATAL("cannot open debug file '%s': %s\n",
581 optarg, strerror(errno));
592 if (optarg == NULL) {
594 } else if (strcmp(optarg, "-") == 0) {
597 trace_file = fopen(optarg, "a");
598 if (debug_file == NULL)
599 FATAL("cannot open debug file '%s': %s\n",
600 optarg, strerror(errno));
605 /* Try ... for more ... */
610 epoll_fd = epoll_create1(EPOLL_CLOEXEC);
612 FATAL("cannot create epoll set: %s\n", strerror(errno));
614 /* Setup signal FD and add to epoll set. */
615 sigset_t signal_mask;
616 sigemptyset(&signal_mask);
617 sigaddset(&signal_mask, SIGINT);
618 sigaddset(&signal_mask, SIGTERM);
619 rc = sigprocmask(SIG_BLOCK, &signal_mask, NULL);
621 FATAL("cannot set process signal mask: %s\n", strerror(errno));
623 signal_fd = signalfd(-1, &signal_mask, SFD_NONBLOCK|SFD_CLOEXEC);
625 FATAL("cannot create signalfd: %s\n", strerror(errno));
627 alr_signal = alr_dev_create(epoll_fd, signal_fd, "signal", &alr_signal_io, NULL);
628 if (alr_signal == NULL)
629 FATAL("cannot register signalfd: %s\n", strerror(errno));
633 /* Open control device. */
634 ctl_fd = open(ctl_path, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
636 FATAL("cannot open '%s': %s\n", ctl_path, strerror(errno));
638 /* Get and print interface version. */
639 oal_version = ioctl(ctl_fd, LUSTRE_ACCESS_LOG_IOCTL_VERSION);
641 FATAL("cannot get ofd access log interface version: %s\n", strerror(errno));
643 DEBUG_D(oal_version);
645 /* Get and print device major used for access log devices. */
646 oal_log_major = ioctl(ctl_fd, LUSTRE_ACCESS_LOG_IOCTL_MAJOR);
647 if (oal_log_major < 0)
648 FATAL("cannot get ofd access log major: %s\n", strerror(errno));
650 DEBUG_D(oal_log_major);
652 /* Add control device to epoll set. */
653 alr_ctl = alr_dev_create(epoll_fd, ctl_fd, "control", &alr_ctl_io, NULL);
655 FATAL("cannot register control device: %s\n", strerror(errno));
660 struct epoll_event ev[32];
661 int timeout = (list_info ? 0 : -1);
664 ev_count = epoll_wait(epoll_fd, ev, ARRAY_SIZE(ev), timeout);
666 if (errno == EINTR) /* Signal or timeout. */
669 ERROR("cannot wait on epoll set: %s\n", strerror(errno));
670 exit_status = EXIT_FAILURE;
676 for (i = 0; i < ev_count; i++) {
677 struct alr_dev *ad = ev[i].data.ptr;
678 unsigned int mask = ev[i].events;
680 rc = (*ad->alr_io)(epoll_fd, ad, mask);
682 case ALR_EXIT_FAILURE:
683 exit_status = EXIT_FAILURE;
685 case ALR_EXIT_SUCCESS:
686 exit_status = EXIT_SUCCESS;
690 alr_dev_free(epoll_fd, ad);
697 } while (!list_info);
699 exit_status = EXIT_SUCCESS;
701 assert(oal_log_minor_max < ARRAY_SIZE(alr_log));
703 for (m = 0; m <= oal_log_minor_max; m++) {
704 if (alr_log[m] == NULL)
708 rc = alr_log_info(alr_log[m]);
710 exit_status = EXIT_FAILURE;
713 alr_dev_free(epoll_fd, &alr_log[m]->alr_dev);
716 alr_dev_free(epoll_fd, alr_ctl);
717 alr_dev_free(epoll_fd, alr_signal);
720 DEBUG_D(exit_status);