LU-10278 utils: allow to migrate without direct io

Using direct i/o to copy file contents during migration minimizes
cache interference, but may significatly reduce performance.
Introduce new option -D/--non-direct to lfs migrate/lfs_migrate that
leaves the tradeoff at the discretion of the caller.

Signed-off-by: Daniel Kobras <d.kobras@science-computing.de>
Signed-off-by: Emoly Liu <emoly.liu@intel.com>
Change-Id: I9c2935ff204ea5385bfc38006c5476b956deb6a7
Reviewed-on: https://review.whamcloud.com/30301
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@intel.com>
Reviewed-by: Dmitry Eremin <dmitry.eremin@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/doc/lfs-migrate.1 b/lustre/doc/lfs-migrate.1
index 7a8609d..ebb013a 100644 (file)
--- a/lustre/doc/lfs-migrate.1
+++ b/lustre/doc/lfs-migrate.1
@@ -34,9 +34,10 @@ For the case of OST object migration:
 .B [--stripe-size|-S \fIstripe_size\fR]
 .B [--pool|-p \fIpool_name\fR]
 .B [--ost-list|-o \fIost_indices\fR]
-.B [--block|-b]
-.B [--non-block|-n] \fIfile|directory\fR
 .B [--component-end|-E \fIcomp_end\fR]
+.B [--block|-b]
+.B [--non-block|-n]
+.B [--non-direct|-D] \fIfile|directory\fR
 .br
 Migrate can also be used for OST objects by omitting \fI-m\fR option. In this
 mode, the command has identical options to

diff --git a/lustre/doc/lfs_migrate.1 b/lustre/doc/lfs_migrate.1
index 5be2a14..4390740 100644 (file)
--- a/lustre/doc/lfs_migrate.1
+++ b/lustre/doc/lfs_migrate.1
@@ -5,6 +5,7 @@
 .SH SYNOPSIS
 .B lfs_migrate
 .RB [ --dry-run ]
+.RB [ -D ]
 .RB [ -h ]
 .RB [ --no-rsync | --rsync ]
 .RB [ -q ]
@@ -58,6 +59,9 @@ or OST index of a new file).
 .B \\--dry-run
 Only print the names of files to be migrated.
 .TP
+.B \\-D
+Do not use direct I/O to copy file contents.
+.TP
 .B \\-h
 Display help information.
 .TP

diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate
index c598f06..306326a 100755 (executable)
--- a/lustre/scripts/lfs_migrate
+++ b/lustre/scripts/lfs_migrate
@@ -44,9 +44,10 @@ old_fid_in_set() {
 
 usage() {
     cat -- <<USAGE 1>&2
-usage: lfs_migrate [--dry-run] [-h] [--no-rsync|--rsync] [-q] [-R] [-s]
-                  [-v] [-y] [-0] [FILE|DIR...]
+usage: lfs_migrate [--dry-run] [-D] [-h] [--no-rsync|--rsync] [-q] [-R]
+                  [-s] [-v] [-y] [-0] [FILE|DIR...]
        --dry-run only print the names of files to be migrated
+       -D do not use direct I/O to copy file contents
        -h show this usage message
        --no-rsync do not fall back to rsync mode even if lfs migrate fails
        -q run quietly (don't print filenames or status)
@@ -94,6 +95,7 @@ OPT_PASSTHROUGH=()
 STRIPE_COUNT=""
 STRIPE_SIZE=""
 POOL=""
+LFS_OPT_DIRECTIO=""
 
 # Examine any long options and arguments.  getopts does not support long
 # options, so they must be stripped out and classified as either options
@@ -165,8 +167,9 @@ done
 # Reset the argument list to include only the short options and file names
 set -- "${OPTS[@]}"
 
-while getopts ":hlnqRsvy0" opt $*; do
+while getopts ":DhlnqRsvy0" opt $*; do
     case $opt in
+       D) LFS_OPT_DIRECTIO="-D";;
        h) usage;;
        l) ;; # maintained for backward compatibility
        n) OPT_DRYRUN=true
@@ -375,8 +378,8 @@ lfs_migrate() {
                # first try to migrate via Lustre tools, then fall back to rsync
                if ! $LFS_MIGRATE_RSYNC_MODE; then
                        if $LFS migrate "${OPT_PASSTHROUGH[@]}" ${BLOCK} \
-                          ${stripe_count} ${stripe_size} "$OLDNAME" &> \
-                          /dev/null; then
+                          $LFS_OPT_DIRECTIO ${stripe_count} ${stripe_size} \
+                           "$OLDNAME" &> /dev/null; then
                                $ECHO "done migrate"
                                for link in ${hlinks[*]}; do
                                        add_to_set "$fid" "$link"

diff --git a/lustre/utils/lfs.c b/lustre/utils/lfs.c
index 412cc1a..66c531f 100644 (file)
--- a/lustre/utils/lfs.c
+++ b/lustre/utils/lfs.c
@@ -221,11 +221,13 @@ static inline int lfs_mirror_split(int argc, char **argv)
        SSM_CMD_COMMON("migrate  ")                                     \
        "                 [--block|-b]\n"                               \
        "                 [--non-block|-n]\n"                           \
+       "                 [--non-direct|-D]\n"                          \
        "                 <filename>\n"                                 \
        SSM_HELP_COMMON                                                 \
        "\n"                                                            \
        "\tblock:        Block file access during data migration (default)\n" \
        "\tnon-block:    Abort migrations if concurrent access is detected\n" \
+       "\tnon-direct:   Do not use direct I/O to copy file contents\n" \
 
 #define SETDIRSTRIPE_USAGE                                     \
        "               [--mdt-count|-c stripe_count>\n"        \
@@ -510,6 +512,7 @@ command_t cmdlist[] = {
         "              [--ost|-o] <ost_indices>\n"
         "              [--block|-b]\n"
         "              [--non-block|-n]\n"
+        "              [--non-direct|-D]\n"
         "              <file|directory>\n"
         "\tstripe_count:     number of OSTs to stripe a file over\n"
         "\tstripe_ost_index: index of the first OST to stripe a file over\n"
@@ -517,7 +520,8 @@ command_t cmdlist[] = {
         "\tpool_name:        name of the predefined pool of OSTs\n"
         "\tost_indices:      OSTs to stripe over, in order\n"
         "\tblock:            wait for the operation to return before continuing\n"
-        "\tnon-block:        do not wait for the operation to return.\n"},
+        "\tnon-block:        do not wait for the operation to return\n"
+        "\tnon-direct:       do not use direct I/O to copy file contents.\n"},
        {"mv", lfs_mv, 0,
         "To move directories between MDTs. This command is deprecated, "
         "use \"migrate\" instead.\n"
@@ -565,17 +569,20 @@ static const char *error_loc = "syserror";
 enum {
        MIGRATION_NONBLOCK      = 1 << 0,
        MIGRATION_MIRROR        = 1 << 1,
+       MIGRATION_NONDIRECT     = 1 << 2,
 };
 
 static int lfs_component_create(char *fname, int open_flags, mode_t open_mode,
                                struct llapi_layout *layout);
 
 static int
-migrate_open_files(const char *name, const struct llapi_stripe_param *param,
+migrate_open_files(const char *name, __u64 migration_flags,
+                  const struct llapi_stripe_param *param,
                   struct llapi_layout *layout, int *fd_src, int *fd_tgt)
 {
        int                      fd = -1;
        int                      fdv = -1;
+       int                      rflags;
        int                      mdt_index;
        int                      random_value;
        char                     parent[PATH_MAX];
@@ -612,7 +619,10 @@ migrate_open_files(const char *name, const struct llapi_stripe_param *param,
        /* open file, direct io */
        /* even if the file is only read, WR mode is nedeed to allow
         * layout swap on fd */
-       fd = open(name, O_RDWR | O_DIRECT);
+       rflags = O_RDWR;
+       if (!(migration_flags & MIGRATION_NONDIRECT))
+               rflags |= O_DIRECT;
+       fd = open(name, rflags);
        if (fd < 0) {
                rc = -errno;
                error_loc = "cannot open source file";
@@ -1016,7 +1026,8 @@ static int lfs_migrate(char *name, __u64 migration_flags,
        int fdv = -1;
        int rc;
 
-       rc = migrate_open_files(name, param, layout, &fd, &fdv);
+       rc = migrate_open_files(name, migration_flags, param, layout,
+                               &fd, &fdv);
        if (rc < 0)
                goto out;
 
@@ -1561,7 +1572,7 @@ static int mirror_extend_layout(char *name, struct llapi_layout *layout)
        int fdv = -1;
        int rc;
 
-       rc = migrate_open_files(name, NULL, layout, &fd, &fdv);
+       rc = migrate_open_files(name, 0, NULL, layout, &fd, &fdv);
        if (rc < 0)
                goto out;
 
@@ -2363,6 +2374,8 @@ static int lfs_setstripe_internal(int argc, char **argv,
 /* find        { .val = 'C',   .name = "ctime",        .has_arg = required_argument }*/
        { .val = 'd',   .name = "delete",       .has_arg = no_argument},
        { .val = 'd',   .name = "destroy",      .has_arg = no_argument},
+       /* --non-direct is only valid in migrate mode */
+       { .val = 'D',   .name = "non-direct",   .has_arg = no_argument },
        { .val = 'E',   .name = "comp-end",     .has_arg = required_argument},
        { .val = 'E',   .name = "component-end",
                                                .has_arg = required_argument},
@@ -2410,7 +2423,7 @@ static int lfs_setstripe_internal(int argc, char **argv,
 
        snprintf(cmd, sizeof(cmd), "%s %s", progname, argv[0]);
        progname = cmd;
-       while ((c = getopt_long(argc, argv, "bc:dE:f:i:I:m:N::no:p:L:s:S:v",
+       while ((c = getopt_long(argc, argv, "bc:dDE:f:i:I:m:N::no:p:L:s:S:v",
                                long_opts, NULL)) >= 0) {
                switch (c) {
                case 0:
@@ -2528,6 +2541,16 @@ static int lfs_setstripe_internal(int argc, char **argv,
                                mirror_flags |= MF_DESTROY;
                        }
                        break;
+               case 'D':
+                       if (!migrate_mode) {
+                               fprintf(stderr,
+                                       "%s %s: -D|--non-direct is valid "
+                                       "only for migrate command\n",
+                                       progname, argv[0]);
+                               goto usage_error;
+                       }
+                       migration_flags |= MIGRATION_NONDIRECT;
+                       break;
                case 'E':
                        if (lsa.lsa_comp_end != 0) {
                                result = comp_args_to_layout(lpp, &lsa);