From adfb154348c742b073a2b8120f2a9bf4c67fb7fa Mon Sep 17 00:00:00 2001 From: Daniel Kobras Date: Wed, 29 Nov 2017 00:26:00 +0800 Subject: [PATCH] LU-10278 utils: allow to migrate without direct io Using direct i/o to copy file contents during migration minimizes cache interference, but may significatly reduce performance. Introduce new option -D/--non-direct to lfs migrate/lfs_migrate that leaves the tradeoff at the discretion of the caller. Signed-off-by: Daniel Kobras Signed-off-by: Emoly Liu Change-Id: I9c2935ff204ea5385bfc38006c5476b956deb6a7 Reviewed-on: https://review.whamcloud.com/30301 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Jinshan Xiong Reviewed-by: Dmitry Eremin Reviewed-by: Oleg Drokin --- lustre/doc/lfs-migrate.1 | 5 +++-- lustre/doc/lfs_migrate.1 | 4 ++++ lustre/scripts/lfs_migrate | 13 ++++++++----- lustre/utils/lfs.c | 35 +++++++++++++++++++++++++++++------ 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/lustre/doc/lfs-migrate.1 b/lustre/doc/lfs-migrate.1 index 7a8609d..ebb013a 100644 --- a/lustre/doc/lfs-migrate.1 +++ b/lustre/doc/lfs-migrate.1 @@ -34,9 +34,10 @@ For the case of OST object migration: .B [--stripe-size|-S \fIstripe_size\fR] .B [--pool|-p \fIpool_name\fR] .B [--ost-list|-o \fIost_indices\fR] -.B [--block|-b] -.B [--non-block|-n] \fIfile|directory\fR .B [--component-end|-E \fIcomp_end\fR] +.B [--block|-b] +.B [--non-block|-n] +.B [--non-direct|-D] \fIfile|directory\fR .br Migrate can also be used for OST objects by omitting \fI-m\fR option. In this mode, the command has identical options to diff --git a/lustre/doc/lfs_migrate.1 b/lustre/doc/lfs_migrate.1 index 5be2a14..4390740 100644 --- a/lustre/doc/lfs_migrate.1 +++ b/lustre/doc/lfs_migrate.1 @@ -5,6 +5,7 @@ .SH SYNOPSIS .B lfs_migrate .RB [ --dry-run ] +.RB [ -D ] .RB [ -h ] .RB [ --no-rsync | --rsync ] .RB [ -q ] @@ -58,6 +59,9 @@ or OST index of a new file). .B \\--dry-run Only print the names of files to be migrated. .TP +.B \\-D +Do not use direct I/O to copy file contents. +.TP .B \\-h Display help information. .TP diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate index c598f06..306326a 100755 --- a/lustre/scripts/lfs_migrate +++ b/lustre/scripts/lfs_migrate @@ -44,9 +44,10 @@ old_fid_in_set() { usage() { cat -- <&2 -usage: lfs_migrate [--dry-run] [-h] [--no-rsync|--rsync] [-q] [-R] [-s] - [-v] [-y] [-0] [FILE|DIR...] +usage: lfs_migrate [--dry-run] [-D] [-h] [--no-rsync|--rsync] [-q] [-R] + [-s] [-v] [-y] [-0] [FILE|DIR...] --dry-run only print the names of files to be migrated + -D do not use direct I/O to copy file contents -h show this usage message --no-rsync do not fall back to rsync mode even if lfs migrate fails -q run quietly (don't print filenames or status) @@ -94,6 +95,7 @@ OPT_PASSTHROUGH=() STRIPE_COUNT="" STRIPE_SIZE="" POOL="" +LFS_OPT_DIRECTIO="" # Examine any long options and arguments. getopts does not support long # options, so they must be stripped out and classified as either options @@ -165,8 +167,9 @@ done # Reset the argument list to include only the short options and file names set -- "${OPTS[@]}" -while getopts ":hlnqRsvy0" opt $*; do +while getopts ":DhlnqRsvy0" opt $*; do case $opt in + D) LFS_OPT_DIRECTIO="-D";; h) usage;; l) ;; # maintained for backward compatibility n) OPT_DRYRUN=true @@ -375,8 +378,8 @@ lfs_migrate() { # first try to migrate via Lustre tools, then fall back to rsync if ! $LFS_MIGRATE_RSYNC_MODE; then if $LFS migrate "${OPT_PASSTHROUGH[@]}" ${BLOCK} \ - ${stripe_count} ${stripe_size} "$OLDNAME" &> \ - /dev/null; then + $LFS_OPT_DIRECTIO ${stripe_count} ${stripe_size} \ + "$OLDNAME" &> /dev/null; then $ECHO "done migrate" for link in ${hlinks[*]}; do add_to_set "$fid" "$link" diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c index 412cc1a..66c531f 100644 --- a/lustre/utils/lfs.c +++ b/lustre/utils/lfs.c @@ -221,11 +221,13 @@ static inline int lfs_mirror_split(int argc, char **argv) SSM_CMD_COMMON("migrate ") \ " [--block|-b]\n" \ " [--non-block|-n]\n" \ + " [--non-direct|-D]\n" \ " \n" \ SSM_HELP_COMMON \ "\n" \ "\tblock: Block file access during data migration (default)\n" \ "\tnon-block: Abort migrations if concurrent access is detected\n" \ + "\tnon-direct: Do not use direct I/O to copy file contents\n" \ #define SETDIRSTRIPE_USAGE \ " [--mdt-count|-c stripe_count>\n" \ @@ -510,6 +512,7 @@ command_t cmdlist[] = { " [--ost|-o] \n" " [--block|-b]\n" " [--non-block|-n]\n" + " [--non-direct|-D]\n" " \n" "\tstripe_count: number of OSTs to stripe a file over\n" "\tstripe_ost_index: index of the first OST to stripe a file over\n" @@ -517,7 +520,8 @@ command_t cmdlist[] = { "\tpool_name: name of the predefined pool of OSTs\n" "\tost_indices: OSTs to stripe over, in order\n" "\tblock: wait for the operation to return before continuing\n" - "\tnon-block: do not wait for the operation to return.\n"}, + "\tnon-block: do not wait for the operation to return\n" + "\tnon-direct: do not use direct I/O to copy file contents.\n"}, {"mv", lfs_mv, 0, "To move directories between MDTs. This command is deprecated, " "use \"migrate\" instead.\n" @@ -565,17 +569,20 @@ static const char *error_loc = "syserror"; enum { MIGRATION_NONBLOCK = 1 << 0, MIGRATION_MIRROR = 1 << 1, + MIGRATION_NONDIRECT = 1 << 2, }; static int lfs_component_create(char *fname, int open_flags, mode_t open_mode, struct llapi_layout *layout); static int -migrate_open_files(const char *name, const struct llapi_stripe_param *param, +migrate_open_files(const char *name, __u64 migration_flags, + const struct llapi_stripe_param *param, struct llapi_layout *layout, int *fd_src, int *fd_tgt) { int fd = -1; int fdv = -1; + int rflags; int mdt_index; int random_value; char parent[PATH_MAX]; @@ -612,7 +619,10 @@ migrate_open_files(const char *name, const struct llapi_stripe_param *param, /* open file, direct io */ /* even if the file is only read, WR mode is nedeed to allow * layout swap on fd */ - fd = open(name, O_RDWR | O_DIRECT); + rflags = O_RDWR; + if (!(migration_flags & MIGRATION_NONDIRECT)) + rflags |= O_DIRECT; + fd = open(name, rflags); if (fd < 0) { rc = -errno; error_loc = "cannot open source file"; @@ -1016,7 +1026,8 @@ static int lfs_migrate(char *name, __u64 migration_flags, int fdv = -1; int rc; - rc = migrate_open_files(name, param, layout, &fd, &fdv); + rc = migrate_open_files(name, migration_flags, param, layout, + &fd, &fdv); if (rc < 0) goto out; @@ -1561,7 +1572,7 @@ static int mirror_extend_layout(char *name, struct llapi_layout *layout) int fdv = -1; int rc; - rc = migrate_open_files(name, NULL, layout, &fd, &fdv); + rc = migrate_open_files(name, 0, NULL, layout, &fd, &fdv); if (rc < 0) goto out; @@ -2363,6 +2374,8 @@ static int lfs_setstripe_internal(int argc, char **argv, /* find { .val = 'C', .name = "ctime", .has_arg = required_argument }*/ { .val = 'd', .name = "delete", .has_arg = no_argument}, { .val = 'd', .name = "destroy", .has_arg = no_argument}, + /* --non-direct is only valid in migrate mode */ + { .val = 'D', .name = "non-direct", .has_arg = no_argument }, { .val = 'E', .name = "comp-end", .has_arg = required_argument}, { .val = 'E', .name = "component-end", .has_arg = required_argument}, @@ -2410,7 +2423,7 @@ static int lfs_setstripe_internal(int argc, char **argv, snprintf(cmd, sizeof(cmd), "%s %s", progname, argv[0]); progname = cmd; - while ((c = getopt_long(argc, argv, "bc:dE:f:i:I:m:N::no:p:L:s:S:v", + while ((c = getopt_long(argc, argv, "bc:dDE:f:i:I:m:N::no:p:L:s:S:v", long_opts, NULL)) >= 0) { switch (c) { case 0: @@ -2528,6 +2541,16 @@ static int lfs_setstripe_internal(int argc, char **argv, mirror_flags |= MF_DESTROY; } break; + case 'D': + if (!migrate_mode) { + fprintf(stderr, + "%s %s: -D|--non-direct is valid " + "only for migrate command\n", + progname, argv[0]); + goto usage_error; + } + migration_flags |= MIGRATION_NONDIRECT; + break; case 'E': if (lsa.lsa_comp_end != 0) { result = comp_args_to_layout(lpp, &lsa); -- 1.8.3.1