LU-17490 tests: verify fanotify works for lustre

The fanotify API provides notification and interception of filesystem
events. Here we prepare a small util to monitor open/read/write/close
events of file in a filesystem. Verify it works for lustre
filesystem.

Signed-off-by: Lei Feng <flei@whamcloud.com>
Test-Parameters: trivial
Change-Id: Id57a59bca16133db645e6804024cba9f11d60f1d
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53869
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Alex Deiter
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index fcfa4db..08da540 100644 (file)
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -74,6 +74,7 @@ THETESTS += create_foreign_dir parse_foreign_dir
 THETESTS += check_fallocate splice-test lseek_test expand_truncate_test
 THETESTS += foreign_symlink_striping lov_getstripe_old io_uring_probe
 THETESTS += fadvise_dontneed_helper llapi_root_test aheadmany
+THETESTS += monitor_lustrefs
 
 if LIBAIO
 THETESTS += aiocp

diff --git a/lustre/tests/monitor_lustrefs.c b/lustre/tests/monitor_lustrefs.c
new file mode 100644 (file)
index 0000000..b9fdf36
--- /dev/null
+++ b/lustre/tests/monitor_lustrefs.c
@@ -0,0 +1,172 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <limits.h>
+#include <errno.h>
+#include <string.h>
+#include <libgen.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/fanotify.h>
+
+
+void usage(const char *progname)
+{
+       const char *base, *msg;
+
+       base = strrchr(progname, '/');
+       if (base == NULL)
+               base = progname;
+       else
+               base++;
+
+       fprintf(stderr, "Usage:\n");
+       fprintf(stderr, "  %s LUSTRE_MOUNT_DIR\n", base);
+       fprintf(stderr, "\n");
+
+       msg =
+"Description:\n"
+"  Monitor some file operations on a lustre fs. Report the events as below:\n"
+"    <events>:<lustre_file>:<pid>:[<command>]\n"
+"\n"
+"  <events>      is 1 event or multiple events separated by '&'. For example,\n"
+"                'open', 'write&close'. Currently only these events are\n"
+"                monitored:\n"
+"                  - open\n"
+"                  - close\n"
+"                  - read\n"
+"                  - write\n"
+"  <lustre_file> is the file to be operated.\n"
+"  <pid>         is the process id who operates the file.\n"
+"  <command>     is the command who operates the file. It is reported only if\n"
+"                the process is still running so it can be found from pid.\n"
+"\n";
+       fprintf(stderr, "%s", msg);
+}
+
+void print_event(struct fanotify_event_metadata *metadata)
+{
+       bool first = true;
+       char procfd_path[PATH_MAX], path[PATH_MAX], cmd_file[PATH_MAX];
+       int path_len, cmd_fd, cmd_len;
+
+       // print event type
+       if (metadata->mask & FAN_OPEN) {
+               printf("open");
+               first = false;
+       }
+       if (metadata->mask & FAN_ACCESS) {
+               if (!first)
+                       printf("&");
+               printf("read");
+               first = false;
+       }
+       if (metadata->mask & FAN_MODIFY) {
+               if (!first)
+                       printf("&");
+               printf("write");
+               first = false;
+       }
+       if (metadata->mask & FAN_CLOSE) {
+               if (!first)
+                       printf("&");
+               printf("close");
+               first = false;
+       }
+       printf(":");
+
+       // print the name of the file
+       snprintf(procfd_path, sizeof(procfd_path), "/proc/self/fd/%d",
+                metadata->fd);
+       path_len = readlink(procfd_path, path, sizeof(path) - 1);
+       if (path_len == -1) {
+               fprintf(stderr, "failed to read link target of %s. %d:%s\n",
+                       procfd_path, errno, strerror(errno));
+               exit(EXIT_FAILURE);
+       }
+       path[path_len] = '\0';
+       printf("%s:", path);
+       close(metadata->fd);
+
+       // print the pid
+       printf("%d:", metadata->pid);
+
+       // try to print the cmdline of process
+       snprintf(cmd_file, sizeof(cmd_file), "/proc/%d/cmdline", metadata->pid);
+       cmd_fd = open(cmd_file, O_RDONLY);
+       if (cmd_fd >= 0) {
+               // reuse cmd_file as buffer
+               cmd_len = read(cmd_fd, cmd_file, sizeof(cmd_file) - 1);
+               if (cmd_len > 0) {
+                       cmd_file[cmd_len] = '\0';
+                       printf("%s", cmd_file);
+               }
+               close(cmd_fd);
+       }
+       printf("\n");
+       fflush(stdout);
+}
+
+int main(int argc, char *argv[])
+{
+       int fd, rc;
+       uint32_t mask;
+       struct fanotify_event_metadata buf[256], *metadata;
+       int len;
+
+       if (argc != 2) {
+               usage(argv[0]);
+               exit(EXIT_FAILURE);
+       }
+
+       fd = fanotify_init(FAN_CLASS_CONTENT, O_RDONLY | O_LARGEFILE);
+       if (fd < 0) {
+               fprintf(stderr, "failed to init fanotify. %d:%s\n", errno,
+                       strerror(errno));
+               exit(EXIT_FAILURE);
+       }
+
+       mask = FAN_OPEN | FAN_ACCESS | FAN_MODIFY | FAN_CLOSE;
+       rc = fanotify_mark(fd, FAN_MARK_ADD | FAN_MARK_MOUNT, mask, AT_FDCWD,
+                          argv[1]);
+       if (rc < 0) {
+               fprintf(stderr,
+                       "failed to open watch descriptor on %s. %d:%s\n",
+                       argv[1], errno, strerror(errno));
+               exit(EXIT_FAILURE);
+       }
+
+       while (1) {
+               len = read(fd, (void *)&buf[0], sizeof(buf));
+               if (len < 0 && errno != EAGAIN) {
+                       fprintf(stderr,
+                               "failed to read from fanotiify file descriptor."
+                               " %d:%s",
+                               errno, strerror(errno));
+                       exit(EXIT_FAILURE);
+               }
+               if (len < 0)
+                       break;
+
+               metadata = &buf[0];
+               while (FAN_EVENT_OK(metadata, len)) {
+                       /* Check run-time and compile-time structures match */
+                       if (metadata->vers != FANOTIFY_METADATA_VERSION) {
+                               fprintf(stderr, "Mismatch of fanotify "
+                                       "metadata version.\n");
+                               exit(EXIT_FAILURE);
+                       }
+
+                       if (metadata->fd >= 0) {
+                               print_event(metadata);
+                               close(metadata->fd);
+                       }
+                       metadata = FAN_EVENT_NEXT(metadata, len);
+               }
+       }
+
+       close(fd);
+       return EXIT_SUCCESS;
+}

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index fed7896..fd78e64 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -31971,6 +31971,49 @@ test_850() {
 }
 run_test 850 "lljobstat can parse living and aggregated job_stats"
 
+test_851() {
+       local dir=$DIR/$tdir
+       local file=$dir/f_test_851_$$
+       local report=/tmp/report_test_851_$$
+       local fanotify_prog=monitor_lustrefs
+       local pid
+
+       test_mkdir $dir || error "failed to create dir $dir"
+
+       $fanotify_prog $DIR > $report &
+       pid=$!
+
+       sleep 1
+       if ! kill -0 $pid; then
+               error "failed to start $fanoify_prog"
+       fi
+
+       stack_trap "kill $pid"
+       stack_trap "rm -f $report"
+
+       echo "1234567890" > $file
+       wait_update_cond localhost "stat -c %s $report" "-gt" "0" 30 ||
+               error "fanotify did not report anything after 30 seconds"
+       grep -a -E "open.*:$file:" $report ||
+               error "no open event for writing $file"
+       grep -a -E "write.*:$file:" $report ||
+               error "no write event for writing $file"
+       grep -a -E "close.*:$file:" $report ||
+               error "no close event for writing $file"
+
+       > $report
+       cat $file
+       wait_update_cond localhost "stat -c %s $report" "-gt" "0" 30 ||
+               error "fanotify did not report anything after 30 seconds"
+       grep -a -E "open.*:$file:" $report ||
+               error "no open event for reading $file"
+       grep -a -E "read.*:$file:" $report ||
+               error "no write event for reading $file"
+       grep -a -E "close.*:$file:" $report ||
+               error "no close event for reading $file"
+}
+run_test 851 "fanotify can monitor open/read/write/close events for lustre fs"
+
 #
 # tests that do cleanup/setup should be run at the end
 #