LU-15613 utils: add lfs_migrate --cp option

Since "lfs_migrate" is only copying a single file at a time, there is
no real benefit from using "rsync" as the only copytool.  Add a short
"--cp" option with suitable arguments to complement "--rsync".

Also allow the copy command to be replaced arbitrarily with RSYNC and
RSYNC_OPTS environment variables to be specified by the caller.

Test-Parameters: trivial
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I66c664948d5cc5a7baa5a65550abfaa8d73ebbe5
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49727
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Feng Lei <flei@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/doc/lfs_migrate.1 b/lustre/doc/lfs_migrate.1
index fc9ffab..63b25ff 100644 (file)
--- a/lustre/doc/lfs_migrate.1
+++ b/lustre/doc/lfs_migrate.1
@@ -5,6 +5,7 @@
 .SH SYNOPSIS
 .B lfs_migrate
 .RB [ "-A " [ -C \fI<cap> \fR] [ -M \fI<min_free> \fR] [ -X \fI<max_free> \fR]]
+.RB [ --cp ]
 .RB [ --dry-run | -n ]
 .RB [ --help | -h ]
 .RB [ --no-rsync | --rsync ]
@@ -70,6 +71,12 @@ directory (potentially changing the stripe count, stripe size, OST pool,
 or OST index of a new file).
 .SH OPTIONS
 .TP
+.B --cp
+Force use of
+.BR cp (1)
+to copy files instead of
+.BR rsync (1).
+.TP
 .B \\--dry-run|-n
 Only print the names of files to be migrated.
 .TP

diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate
index c81175f..dbc5ad2 100755 (executable)
--- a/lustre/scripts/lfs_migrate
+++ b/lustre/scripts/lfs_migrate
@@ -12,7 +12,8 @@
 # process, but it would catch the worst cases of in-use files, but
 # to be 100% safe the administrator needs to ensure this is safe.
 
-RSYNC=${RSYNC:-rsync}
+RSYNC=${RSYNC:-"rsync"}
+RSYNC_OPTS=${RSYNC_OPTS:-"-a --inplace"}
 ECHO=echo
 LFS=${LFS:-lfs}
 RSYNC_WITH_HLINKS=false
@@ -44,11 +45,11 @@ old_fid_in_set() {
 
 usage() {
     cat -- <<USAGE 1>&2
-usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
+usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync|--cp]
                   [--auto-stripe|-A [-C <cap>]
                   [--min-free|-M <min_free>] [--max-free|-X <max_free>]]
                   [--pool|-p <pool>] [--stripe-count|-c <stripe_count>]
-                  [--stripe-size|-S <stripe_size>]
+                  [--stripe-size|-S <stripe_size>] [--quiet|-q]
                   [-D] [-h] [-n] [-S]
                   [--restripe|-R] [--skip|-s] [--verbose|-v] [--yes|-y] [-0]
                   [FILE|DIR...]
@@ -58,6 +59,7 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
                   restripe file using the specified <stripe_count>
        -C <cap>   when -A is set, limit the migrated file to use on each OST
                   at most 1/<cap> of the available space of the smallest OST
+       --cp       force the use of 'cp' instead of migrate or rsync
        -D         do not use direct I/O to copy file contents
        -h         show this usage message
        -M <min_free>
@@ -78,12 +80,13 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
                   when -A is set, limit the amount of space on each OST that
                   can be considered available for the migration to
                   <max_free> KB
-       -y         answer 'y' to usage question (only when --rsync used)
+       -y         answer 'y' to usage question (only when --rsync/--cp used)
        -0         input file names on stdin are separated by a null character
 
 Options '-A', and '-R' are mutually exclusive with each other, and any
 specific layout (e.g. any specific parameters like '-c', '-S', '-E', '-p').
 Options '-C', '-M', and '-X' are ignored if '-A' is not set.
+Options --rsync, --cp, --no-rsync may not be specified at the same time.
 
 If a directory is an argument, all files in the directory are migrated.
 If no file/directory is given, the file list is read from standard input.
@@ -139,40 +142,43 @@ OPT_CAP=100
 while [ -n "$*" ]; do
        arg="$1"
        case "$arg" in
-       -h|--help) usage;;
-       -l|--link) ;; # maintained backward compatibility for now
-       -n) OPT_DRYRUN=true; OPT_YES=true
-          echo "$PROG: -n deprecated, use --dry-run or --non-block" 1>&2;;
-       --dry-run) OPT_DRYRUN=true; OPT_YES=true;;
-       -p|--pool) # if within a component, pass through pool
-               $OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_POOL="-p $2";
-               shift;;
-       -q|--quiet) ECHO=:;;
-       -R|--restripe) OPT_RESTRIPE=true;;
-       -s|--skip) OPT_CHECK=false;;
-       -v|--verbose) OPT_DEBUG=true; ECHO=echo;;
-       -y|--yes) OPT_YES=true;;
-       -0) OPT_NULL=true;;
+       --auto-stripe|-A) OPT_AUTOSTRIPE=true;;
        -b|--block|--non-block|--non-direct|-D|--no-verify)
           # Always pass non-layout options to 'lfs migrate'
           OPT_PASSTHROUGH+=("$arg");;
-       --rsync) OPT_RSYNC=true; OPT_NO_RSYNC=false;;
-       --no-rsync) OPT_NO_RSYNC=true; OPT_RSYNC=false;;
+       --cp) $OPT_RSYNC && [[ "$RSYNC" != "cp" ]] && OPT_NO_RSYNC=true
+          OPT_RSYNC=true; RSYNC="cp"; RSYNC_OPTS="-p";;
        --copy|--yaml|--file) OPT_COPY=true;
           # these options have files as arguments, pass both through
           OPT_LAYOUT+="$arg $2 "; shift;;
-       --auto-stripe|-A) OPT_AUTOSTRIPE=true;;
+       --dry-run) OPT_DRYRUN=true; OPT_YES=true;;
+       -c|--stripe-count) # if within a component, pass through stripe_count
+               $OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_STRIPE_COUNT="$2"
+               shift;;
        -C) OPT_CAP="$2"; shift;;
        -D) LFS_OPT_DIRECTIO="-D";;
        -E|--comp-end|--component-end) OPT_COMP=true; OPT_LAYOUT+="$arg ";;
+       -h|--help) usage;;
+       -l|--link) ;; # maintained backward compatibility for now
        -M|--min-free) OPT_MINFREE="$2"; shift;;
-       -X|--max-free) OPT_MAXFREE="$2"; shift;;
-       -c|--stripe-count) # if within a component, pass through stripe_count
-               $OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_STRIPE_COUNT="$2"
+       -n) OPT_DRYRUN=true; OPT_YES=true
+          echo "$PROG: -n deprecated, use --dry-run or --non-block" 1>&2;;
+       -p|--pool) # if within a component, pass through pool
+               $OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_POOL="-p $2";
                shift;;
+       -q|--quiet) ECHO=:;;
+       -R|--restripe) OPT_RESTRIPE=true;;
+       --no-rsync) OPT_NO_RSYNC=true; OPT_RSYNC=false;;
+       --rsync) [[ "$RSYNC" != "rsync" ]] && OPT_NO_RSYNC=true || OPT_NO_RSYNC=false
+          OPT_RSYNC=true ;;
+       -s|--skip) OPT_CHECK=false;;
        -S|--stripe-size) # if within a component, pass through stripe_size
                $OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_STRIPE_SIZE="$2"
                shift;;
+       -v|--verbose) OPT_DEBUG=true; ECHO=echo;;
+       -X|--max-free) OPT_MAXFREE="$2"; shift;;
+       -y|--yes) OPT_YES=true;;
+       -0) OPT_NULL=true;;
        *) # Pass other non-file layout options to 'lfs migrate'
           [[ -e "$arg" ]] && OPT_FILE+="$arg " && break || OPT_LAYOUT+="$arg "
        esac
@@ -191,6 +197,12 @@ elif $OPT_RESTRIPE && $OPT_AUTOSTRIPE; then
        exit 1
 fi
 
+if $OPT_RSYNC && $OPT_NO_RSYNC; then
+       echo "$PROG: Options --cp, --rsync, and --no-rsync may not be" \
+               "specified at the same time." 1>&2
+       exit 1
+fi
+
 if $OPT_RSYNC && ! $OPT_YES; then
        echo ""
        echo "'lfs_migrate --rsync' is NOT SAFE for moving in-use files." 1>&2
@@ -202,8 +214,8 @@ if $OPT_RSYNC && ! $OPT_YES; then
 fi
 
 # if rsync has --xattr support, then try to copy the xattrs.
-$RSYNC --help 2>&1 | grep -q xattr && RSYNC_OPTS="$RSYNC_OPTS -X"
-$RSYNC --help 2>&1 | grep -q acls && RSYNC_OPTS="$RSYNC_OPTS -A"
+$RSYNC --help 2>&1 | grep -q -- --xattrs && RSYNC_OPTS="$RSYNC_OPTS --xattrs"
+$RSYNC --help 2>&1 | grep -q -- --acls && RSYNC_OPTS="$RSYNC_OPTS --acls"
 # If rsync copies lustre xattrs in the future, then we can skip lfs (bug 22189)
 strings $(which $RSYNC) 2>&1 | grep -q lustre && LFS=:
 
@@ -461,8 +473,8 @@ lfs_migrate() {
                        continue
                fi
 
-               if ! $OPT_COPY && ! $OPT_COMP &&
-                  [[ ${layout_info[$l_comp_count]} > 0 ]]; then
+               if ! $OPT_COPY && ! $OPT_COMP && ! $OPT_RSYNC &&
+                  (( ${layout_info[$l_comp_count]} > 0 )); then
                        layout+="--copy $OLDNAME"
                        OPT_COPY=true
                fi
@@ -501,6 +513,8 @@ lfs_migrate() {
                                echo -e "\n$LFS migrate $layout \"$OLDNAME\""
                        fi
                        if $LFS migrate $layout "$OLDNAME"; then
+                               (( $mirror_count > 1 )) &&
+                                       $LFS mirror resync "$OLDNAME"
                                $ECHO "done"
                                # no-op if hlinks empty for 1-link files
                                for link in ${hlinks[*]}; do
@@ -531,7 +545,7 @@ lfs_migrate() {
                fi
 
                # we use --inplace, since we created our own temp file already
-               if ! $RSYNC -a --inplace $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
+               if ! $RSYNC $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
                        echo -e "\r$OLDNAME: copy error, exiting" 1>&2
                        exit 4
                fi
@@ -546,7 +560,8 @@ lfs_migrate() {
                        exit 12
                fi
 
-               $ECHO "done rsync"
+               (( $mirror_count > 1 )) && $LFS mirror resync "$OLDNAME"
+               $ECHO "done $RSYNC"
                # no-op if hlinks empty for 1-link files
                for link in ${hlinks[*]}; do
                        if [ "$link" != "$OLDNAME" ]; then