From 7101742b4590542e9e6d1590d0ac5e692109f9da Mon Sep 17 00:00:00 2001 From: Lei Feng Date: Wed, 31 Jan 2024 16:39:55 +0800 Subject: [PATCH] LU-17490 tests: verify fanotify works for lustre The fanotify API provides notification and interception of filesystem events. Here we prepare a small util to monitor open/read/write/close events of file in a filesystem. Verify it works for lustre filesystem. Signed-off-by: Lei Feng Test-Parameters: trivial Change-Id: Id57a59bca16133db645e6804024cba9f11d60f1d Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53869 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Alex Deiter Reviewed-by: Oleg Drokin --- lustre/tests/Makefile.am | 1 + lustre/tests/monitor_lustrefs.c | 172 ++++++++++++++++++++++++++++++++++++++++ lustre/tests/sanity.sh | 43 ++++++++++ 3 files changed, 216 insertions(+) create mode 100644 lustre/tests/monitor_lustrefs.c diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index fcfa4db..08da540 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -74,6 +74,7 @@ THETESTS += create_foreign_dir parse_foreign_dir THETESTS += check_fallocate splice-test lseek_test expand_truncate_test THETESTS += foreign_symlink_striping lov_getstripe_old io_uring_probe THETESTS += fadvise_dontneed_helper llapi_root_test aheadmany +THETESTS += monitor_lustrefs if LIBAIO THETESTS += aiocp diff --git a/lustre/tests/monitor_lustrefs.c b/lustre/tests/monitor_lustrefs.c new file mode 100644 index 0000000..b9fdf364 --- /dev/null +++ b/lustre/tests/monitor_lustrefs.c @@ -0,0 +1,172 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +void usage(const char *progname) +{ + const char *base, *msg; + + base = strrchr(progname, '/'); + if (base == NULL) + base = progname; + else + base++; + + fprintf(stderr, "Usage:\n"); + fprintf(stderr, " %s LUSTRE_MOUNT_DIR\n", base); + fprintf(stderr, "\n"); + + msg = +"Description:\n" +" Monitor some file operations on a lustre fs. Report the events as below:\n" +" :::[]\n" +"\n" +" is 1 event or multiple events separated by '&'. For example,\n" +" 'open', 'write&close'. Currently only these events are\n" +" monitored:\n" +" - open\n" +" - close\n" +" - read\n" +" - write\n" +" is the file to be operated.\n" +" is the process id who operates the file.\n" +" is the command who operates the file. It is reported only if\n" +" the process is still running so it can be found from pid.\n" +"\n"; + fprintf(stderr, "%s", msg); +} + +void print_event(struct fanotify_event_metadata *metadata) +{ + bool first = true; + char procfd_path[PATH_MAX], path[PATH_MAX], cmd_file[PATH_MAX]; + int path_len, cmd_fd, cmd_len; + + // print event type + if (metadata->mask & FAN_OPEN) { + printf("open"); + first = false; + } + if (metadata->mask & FAN_ACCESS) { + if (!first) + printf("&"); + printf("read"); + first = false; + } + if (metadata->mask & FAN_MODIFY) { + if (!first) + printf("&"); + printf("write"); + first = false; + } + if (metadata->mask & FAN_CLOSE) { + if (!first) + printf("&"); + printf("close"); + first = false; + } + printf(":"); + + // print the name of the file + snprintf(procfd_path, sizeof(procfd_path), "/proc/self/fd/%d", + metadata->fd); + path_len = readlink(procfd_path, path, sizeof(path) - 1); + if (path_len == -1) { + fprintf(stderr, "failed to read link target of %s. %d:%s\n", + procfd_path, errno, strerror(errno)); + exit(EXIT_FAILURE); + } + path[path_len] = '\0'; + printf("%s:", path); + close(metadata->fd); + + // print the pid + printf("%d:", metadata->pid); + + // try to print the cmdline of process + snprintf(cmd_file, sizeof(cmd_file), "/proc/%d/cmdline", metadata->pid); + cmd_fd = open(cmd_file, O_RDONLY); + if (cmd_fd >= 0) { + // reuse cmd_file as buffer + cmd_len = read(cmd_fd, cmd_file, sizeof(cmd_file) - 1); + if (cmd_len > 0) { + cmd_file[cmd_len] = '\0'; + printf("%s", cmd_file); + } + close(cmd_fd); + } + printf("\n"); + fflush(stdout); +} + +int main(int argc, char *argv[]) +{ + int fd, rc; + uint32_t mask; + struct fanotify_event_metadata buf[256], *metadata; + int len; + + if (argc != 2) { + usage(argv[0]); + exit(EXIT_FAILURE); + } + + fd = fanotify_init(FAN_CLASS_CONTENT, O_RDONLY | O_LARGEFILE); + if (fd < 0) { + fprintf(stderr, "failed to init fanotify. %d:%s\n", errno, + strerror(errno)); + exit(EXIT_FAILURE); + } + + mask = FAN_OPEN | FAN_ACCESS | FAN_MODIFY | FAN_CLOSE; + rc = fanotify_mark(fd, FAN_MARK_ADD | FAN_MARK_MOUNT, mask, AT_FDCWD, + argv[1]); + if (rc < 0) { + fprintf(stderr, + "failed to open watch descriptor on %s. %d:%s\n", + argv[1], errno, strerror(errno)); + exit(EXIT_FAILURE); + } + + while (1) { + len = read(fd, (void *)&buf[0], sizeof(buf)); + if (len < 0 && errno != EAGAIN) { + fprintf(stderr, + "failed to read from fanotiify file descriptor." + " %d:%s", + errno, strerror(errno)); + exit(EXIT_FAILURE); + } + if (len < 0) + break; + + metadata = &buf[0]; + while (FAN_EVENT_OK(metadata, len)) { + /* Check run-time and compile-time structures match */ + if (metadata->vers != FANOTIFY_METADATA_VERSION) { + fprintf(stderr, "Mismatch of fanotify " + "metadata version.\n"); + exit(EXIT_FAILURE); + } + + if (metadata->fd >= 0) { + print_event(metadata); + close(metadata->fd); + } + metadata = FAN_EVENT_NEXT(metadata, len); + } + } + + close(fd); + return EXIT_SUCCESS; +} diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index fed7896..fd78e64 100755 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -31971,6 +31971,49 @@ test_850() { } run_test 850 "lljobstat can parse living and aggregated job_stats" +test_851() { + local dir=$DIR/$tdir + local file=$dir/f_test_851_$$ + local report=/tmp/report_test_851_$$ + local fanotify_prog=monitor_lustrefs + local pid + + test_mkdir $dir || error "failed to create dir $dir" + + $fanotify_prog $DIR > $report & + pid=$! + + sleep 1 + if ! kill -0 $pid; then + error "failed to start $fanoify_prog" + fi + + stack_trap "kill $pid" + stack_trap "rm -f $report" + + echo "1234567890" > $file + wait_update_cond localhost "stat -c %s $report" "-gt" "0" 30 || + error "fanotify did not report anything after 30 seconds" + grep -a -E "open.*:$file:" $report || + error "no open event for writing $file" + grep -a -E "write.*:$file:" $report || + error "no write event for writing $file" + grep -a -E "close.*:$file:" $report || + error "no close event for writing $file" + + > $report + cat $file + wait_update_cond localhost "stat -c %s $report" "-gt" "0" 30 || + error "fanotify did not report anything after 30 seconds" + grep -a -E "open.*:$file:" $report || + error "no open event for reading $file" + grep -a -E "read.*:$file:" $report || + error "no write event for reading $file" + grep -a -E "close.*:$file:" $report || + error "no close event for reading $file" +} +run_test 851 "fanotify can monitor open/read/write/close events for lustre fs" + # # tests that do cleanup/setup should be run at the end # -- 1.8.3.1