From 7e613cdf228749fa8d2b8cba98f37deccd7c9d39 Mon Sep 17 00:00:00 2001 From: "John L. Hammond" Date: Tue, 1 Mar 2022 14:16:52 -0600 Subject: [PATCH] EX-4015 lipe: use direct IO Use direct IO by default in lipe_scan3. Retry ext2fs_open() without EXT2_FLAG_DIRECT_IO if it fails. Add a --direct-io=0|1 option to explicitly disable or enable direct IO. Add an --io-options option to pass down ext2 io_manager options. Test-Parameters: trivial testlist=sanity-lipe-find3 serverextra_install_params="--packages lipe-scan" Test-Parameters: trivial testlist=sanity-lipe-scan3 serverextra_install_params="--packages lipe-scan" facet=mds1 Signed-off-by: John L. Hammond Change-Id: I25347949bbff9e697da26431807daf37cfb720fa Reviewed-on: https://review.whamcloud.com/46682 Tested-by: jenkins --- lipe/src/lipe_scan3/ls3_main.c | 17 ++++++++++++++--- lipe/src/lipe_scan3/ls3_scan.c | 42 ++++++++++++++++++++++++++++++------------ lipe/src/lipe_scan3/ls3_scan.h | 4 ++++ 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/lipe/src/lipe_scan3/ls3_main.c b/lipe/src/lipe_scan3/ls3_main.c index 5b77536..6f365fc 100644 --- a/lipe/src/lipe_scan3/ls3_main.c +++ b/lipe/src/lipe_scan3/ls3_main.c @@ -45,7 +45,8 @@ static const char *ls3_device_path; static const char *ls3_client_mount_path = LS3_CSTR_AUTO; static int ls3_thread_count; static enum ls3_object_attr_bit ls3_required_attrs = LS3_OBJECT_ATTR_DEFAULT; - +int ls3_direct_io = 2; +const char *ls3_io_options = ""; static SCM ls3_scm_fid_vtable = SCM_UNDEFINED; SCM ls3_scm_policy_exception_handler; @@ -1077,13 +1078,14 @@ static void help(void) " --null use a zero byte (the ASCII NUL character) to delimit output of --print-*\n" " --client-mount=MOUNT use the Lustre client at MOUNT for FID to path\n" " --no-client-mount disable automatic client mount detection\n" +" --direct-io=0|1 disable or enable use of direct IO\n" +" --io-options=OPTIONS pass OPTIONS to ext2fs_open2()\n" " --required-attrs=ATTR... only apply policy to inodes with ATTR...\n" " (use '--list-attrs' to see available attributes)\n" " --thread-count=COUNT use COUNT scanning threads\n" " (default is number of processors / 4)\n" " --list-attrs list available attribute names and exit\n" " --list-json-attrs list available JSON attribute names and exit\n" - " --debug remove insects from device\n" " -l, --load=FILE read and evaluate FILE before scanning\n" " multiple --load options may be used\n" @@ -1098,6 +1100,8 @@ enum { LS3_OPT_ALL_PATHS = 1, LS3_OPT_CLIENT_MOUNT, LS3_OPT_DEBUG, + LS3_OPT_DIRECT_IO, + LS3_OPT_IO_OPTIONS, LS3_OPT_LIST_ATTRS, LS3_OPT_LIST_JSON_ATTRS, LS3_OPT_NO_CLIENT_MOUNT, @@ -1120,6 +1124,8 @@ enum { static struct option options[] = { { "client-mount", required_argument, NULL, LS3_OPT_CLIENT_MOUNT }, { "debug", no_argument, NULL, LS3_OPT_DEBUG }, + { "direct-io", required_argument, NULL, LS3_OPT_DIRECT_IO }, + { "io-options", required_argument, NULL, LS3_OPT_IO_OPTIONS }, { "list-attrs", no_argument, NULL, LS3_OPT_LIST_ATTRS }, { "list-json-attrs", no_argument, NULL, LS3_OPT_LIST_JSON_ATTRS }, { "no-client-mount", no_argument, NULL, LS3_OPT_NO_CLIENT_MOUNT }, @@ -1132,7 +1138,6 @@ static struct option options[] = { { "all-paths", no_argument, NULL, LS3_OPT_ALL_PATHS }, { "required-attrs", required_argument, NULL, LS3_OPT_REQUIRED_ATTRS }, { "thread-count", required_argument, NULL, LS3_OPT_THREAD_COUNT }, - { "help", no_argument, NULL, LS3_OPT_HELP }, { "interactive", no_argument, NULL, LS3_OPT_INTERACTIVE }, { "load", required_argument, NULL, LS3_OPT_LOAD }, @@ -1288,6 +1293,12 @@ static void ls3_main_scm(void *data, int argc, char *argv[]) case LS3_OPT_DEBUG: ls3_debug = true; break; + case LS3_OPT_DIRECT_IO: + ls3_direct_io = atoi(optarg); + break; + case LS3_OPT_IO_OPTIONS: + ls3_io_options = optarg; + break; case LS3_OPT_LIST_ATTRS: ls3_list_attrs(); exit(EXIT_SUCCESS); diff --git a/lipe/src/lipe_scan3/ls3_scan.c b/lipe/src/lipe_scan3/ls3_scan.c index 614d623..3a593a6 100644 --- a/lipe/src/lipe_scan3/ls3_scan.c +++ b/lipe/src/lipe_scan3/ls3_scan.c @@ -788,8 +788,13 @@ static void *ls3_scan_thread_start_scm(void *arg) goto out; } - rc = ext2fs_open(dev, EXT2_FLAG_SOFTSUPP_FEATURES, - 0, 0, unix_io_manager, &fs); + rc = ext2fs_open2(dev, + li->li_io_options, + li->li_ext2_flags, + 0, /* superblock */ + 0, /* block_size */ + unix_io_manager, + &fs); if (rc != 0) { LS3_ERROR("cannot open backing filesystem in '%s': %s\n", dev, error_message(rc)); @@ -1194,6 +1199,11 @@ int ls3_scan(const char *device_path, .li_scm_client_mount_fd = scm_from_int(-1), .li_scm_policy_thunk = policy_thunk, .li_scm_thread_count = scm_from_int(0), + .li_ext2_flags = EXT2_FLAG_64BITS | + EXT2_FLAG_SKIP_MMP | + EXT2_FLAG_IGNORE_SB_ERRORS | + (ls3_direct_io ? EXT2_FLAG_DIRECT_IO : 0), + .li_io_options = ls3_io_options, }; struct lustre_disk_data ldd; ext2_filsys fs = NULL; @@ -1205,17 +1215,25 @@ int ls3_scan(const char *device_path, LS3_DEBUG_S(client_mount_path); LS3_DEBUG_X(required_attrs); LS3_DEBUG_D(thread_count); - - rc = ext2fs_open(device_path, - EXT2_FLAG_64BITS | - EXT2_FLAG_SKIP_MMP | - EXT2_FLAG_IGNORE_SB_ERRORS | - EXT2_FLAG_SUPER_ONLY, - 0 /* superblock */, - 0 /* block_size */, - unix_io_manager, - &fs); +retry_open: + LS3_DEBUG_X(li.li_ext2_flags); + LS3_DEBUG_X(li.li_io_options); + rc = ext2fs_open2(device_path, + li.li_io_options, + li.li_ext2_flags | EXT2_FLAG_SUPER_ONLY, + 0 /* superblock */, + 0 /* block_size */, + unix_io_manager, + &fs); + LS3_DEBUG_D(rc); if (rc != 0) { + if (ls3_direct_io == 2) { + /* Direct IO fails in some versions of e2fsprogs. See LU-15590. */ + ls3_direct_io = 0; + li.li_ext2_flags &= ~EXT2_FLAG_DIRECT_IO; + goto retry_open; + } + LS3_ERROR("cannot open backing filesystem '%s': %s\n", device_path, error_message(rc)); rc = LS3_EXIT_SCAN_ERROR; diff --git a/lipe/src/lipe_scan3/ls3_scan.h b/lipe/src/lipe_scan3/ls3_scan.h index 1abab18..b32fe3a 100644 --- a/lipe/src/lipe_scan3/ls3_scan.h +++ b/lipe/src/lipe_scan3/ls3_scan.h @@ -37,6 +37,8 @@ struct ls3_instance { char *li_client_mount_path; int li_client_mount_fd; enum ls3_object_attr_bit li_required_attrs; + unsigned int li_ext2_flags; + const char *li_io_options; SCM li_scm_policy_thunk; SCM li_scm_device_path; SCM li_scm_device_name; @@ -54,6 +56,8 @@ struct ls3_context { struct ls3_object_attrs *lc_attrs; }; +extern int ls3_direct_io; +extern const char *ls3_io_options; extern const char LS3_CSTR_AUTO[]; extern const char LS3_CSTR_NONE[]; -- 1.8.3.1