LU-15613 utils: add lfs_migrate --cp option

Since "lfs_migrate" is only copying a single file at a time, there is
no real benefit from using "rsync" as the only copytool.  Add a short
"--cp" option with suitable arguments to complement "--rsync".

Also allow the copy command to be replaced arbitrarily with RSYNC and
RSYNC_OPTS environment variables to be specified by the caller.

Test-Parameters: trivial
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I66c664948d5cc5a7baa5a65550abfaa8d73ebbe5
Reviewed-on: https://review.whamcloud.com/46691
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>

diff --git a/lustre/doc/lfs_migrate.1 b/lustre/doc/lfs_migrate.1
index c706ceb..21b09f5 100644 (file)
--- a/lustre/doc/lfs_migrate.1
+++ b/lustre/doc/lfs_migrate.1
@@ -5,6 +5,7 @@
 .SH SYNOPSIS
 .B lfs_migrate
 .RB [ "-A " [ -C \fI<cap> \fR] [ -M \fI<min_free> \fR] [ -X \fI<max_free> \fR]]
+.RB [ --cp ]
 .RB [ --dry-run | -n ]
 .RB [ --help | -h ]
 .RB [ --no-rsync | --rsync ]
@@ -70,6 +71,12 @@ directory (potentially changing the stripe count, stripe size, OST pool,
 or OST index of a new file).
 .SH OPTIONS
 .TP
+.B --cp
+Force use of
+.BR cp (1)
+to copy files instead of
+.BR rsync (1).
+.TP
 .B \\--dry-run|-n
 Only print the names of files to be migrated.
 .TP

diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate
index 43ccd60..a480257 100755 (executable)
--- a/lustre/scripts/lfs_migrate
+++ b/lustre/scripts/lfs_migrate
@@ -12,7 +12,8 @@
 # process, but it would catch the worst cases of in-use files, but
 # to be 100% safe the administrator needs to ensure this is safe.
 
-RSYNC=${RSYNC:-rsync}
+RSYNC=${RSYNC:-"rsync"}
+RSYNC_OPTS=${RSYNC_OPTS:-"-a --inplace"}
 OPT_RSYNC=${LFS_MIGRATE_RSYNC_MODE:-false}
 ECHO=echo
 LFS=${LFS:-lfs}
@@ -45,11 +46,11 @@ old_fid_in_set() {
 
 usage() {
     cat -- <<USAGE 1>&2
-usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
+usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync|--cp]
                   [--auto-stripe|-A [-C <cap>]
                   [--min-free|-M <min_free>] [--max-free|-X <max_free>]]
                   [--pool|-p <pool>] [--stripe-count|-c <stripe_count>]
-                  [--stripe-size|-S <stripe_size>]
+                  [--stripe-size|-S <stripe_size>] [--quiet|-q]
                   [-D] [-h] [-n] [-S]
                   [--restripe|-R] [--skip|-s] [--verbose|-v] [--yes|-y] [-0]
                   [FILE|DIR...]
@@ -59,6 +60,7 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
                   restripe file using the specified <stripe_count>
        -C <cap>   when -A is set, limit the migrated file to use on each OST
                   at most 1/<cap> of the available space of the smallest OST
+       --cp       force the use of 'cp' instead of migrate or rsync
        -D         do not use direct I/O to copy file contents
        -h         show this usage message
        -M <min_free>
@@ -84,8 +86,7 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
 
 Options '-A', '-c', and '-R' are mutually exclusive.
 Options '-C', '-M', and '-X' are ignored if '-A' is not set.
-
-The --rsync and --no-rsync options may not be specified at the same time.
+Options --rsync, --cp, --no-rsync may not be specified at the same time.
 
 If a directory is an argument, all files in the directory are migrated.
 If no file/directory is given, the file list is read from standard input.
@@ -134,32 +135,35 @@ OPT_CAP=100
 while [ -n "$*" ]; do
        arg="$1"
        case "$arg" in
+       --auto-stripe|-A) OPT_AUTOSTRIPE=true;;
+       -b|--block|--non-block|--non-direct|-D|--no-verify)
+          # Always pass non-layout options to 'lfs migrate'
+          OPT_PASSTHROUGH+=("$arg");;
+       --cp) $OPT_RSYNC && [[ "$RSYNC" != "cp" ]] && OPT_NO_RSYNC=true
+          OPT_RSYNC=true; RSYNC="cp"; RSYNC_OPTS="-p";;
+       --copy|--yaml|--file) OPT_COMP=true;
+          # these options have files as arguments, pass both through
+          OPT_LAYOUT+="$arg $2 "; shift;;
+       --dry-run) OPT_DRYRUN=true; OPT_YES=true;;
+       -c|--stripe-count) OPT_STRIPE_COUNT="$2"; shift;;
+       -C) OPT_CAP="$2"; shift;;
        -h|--help) usage;;
        -l|--link) ;; # maintained backward compatibility for now
+       -M|--min-free) OPT_MINFREE="$2"; shift;;
        -n) OPT_DRYRUN=true; OPT_YES=true
           echo "$PROG: -n deprecated, use --dry-run or --non-block" 1>&2;;
-       --dry-run) OPT_DRYRUN=true; OPT_YES=true;;
        -p|--pool) OPT_POOL="$arg $2"; OPT_LAYOUT+="$OPT_POOL "; shift;;
        -q|--quiet) ECHO=:;;
        -R|--restripe) OPT_RESTRIPE=true;;
+       --no-rsync) OPT_NO_RSYNC=true;;
+       --rsync) $OPT_RSYNC && [[ "$RSYNC" != "rsync" ]] && OPT_NO_RSYNC=true
+          OPT_RSYNC=true ;;
        -s|--skip) OPT_CHECK=false;;
+       -S|--stripe-size) OPT_STRIPE_SIZE="$2"; shift;;
        -v|--verbose) OPT_DEBUG=true; ECHO=echo;;
+       -X|--max-free) OPT_MAXFREE="$2"; shift;;
        -y|--yes) OPT_YES=true;;
        -0) OPT_NULL=true;;
-       -b|--block|--non-block|--non-direct|-D|--no-verify)
-          # Always pass non-layout options to 'lfs migrate'
-          OPT_PASSTHROUGH+=("$arg");;
-       --rsync) OPT_RSYNC=true;;
-       --no-rsync) OPT_NO_RSYNC=true;;
-       --copy|--yaml|--file) OPT_COMP=true;
-          # these options have files as arguments, pass both through
-          OPT_LAYOUT+="$arg $2 "; shift;;
-       --auto-stripe|-A) OPT_AUTOSTRIPE=true;;
-       -C) OPT_CAP="$2"; shift;;
-       -M|--min-free) OPT_MINFREE="$2"; shift;;
-       -X|--max-free) OPT_MAXFREE="$2"; shift;;
-       -c|--stripe-count) OPT_STRIPE_COUNT="$2"; shift;;
-       -S|--stripe-size) OPT_STRIPE_SIZE="$2"; shift;;
        *) # Pass other non-file layout options to 'lfs migrate'
           [ -e "$arg" ] && OPT_FILE+="$arg " && break || OPT_LAYOUT+="$arg "
        esac
@@ -182,7 +186,7 @@ elif $OPT_AUTOSTRIPE && $OPT_RESTRIPE; then
 fi
 
 if $OPT_RSYNC && $OPT_NO_RSYNC; then
-       echo "$PROG: Options --rsync and --no-rsync may not be" \
+       echo "$PROG: Options --cp, --rsync, and --no-rsync may not be" \
                "specified at the same time." 1>&2
        exit 1
 fi
@@ -202,8 +206,8 @@ if ! $OPT_YES; then
 fi
 
 # if rsync has --xattr support, then try to copy the xattrs.
-$RSYNC --help 2>&1 | grep -q xattr && RSYNC_OPTS="$RSYNC_OPTS -X"
-$RSYNC --help 2>&1 | grep -q acls && RSYNC_OPTS="$RSYNC_OPTS -A"
+$RSYNC --help 2>&1 | grep -q -- --xattrs && RSYNC_OPTS="$RSYNC_OPTS --xattrs"
+$RSYNC --help 2>&1 | grep -q -- --acls && RSYNC_OPTS="$RSYNC_OPTS --acls"
 # If rsync copies lustre xattrs in the future, then we can skip lfs (bug 22189)
 strings $(which $RSYNC) 2>&1 | grep -q lustre && LFS=:
 
@@ -447,7 +451,8 @@ lfs_migrate() {
                        continue
                fi
 
-               if ! $OPT_COMP && [ ${layout_info[$l_comp_count]} -gt 0 ]; then
+               if ! $OPT_COMP && ! $OPT_RSYNC &&
+                  (( ${layout_info[$l_comp_count]} > 0 )); then
                        layout+="--copy $OLDNAME"
                        OPT_COMP=true
                fi
@@ -486,6 +491,8 @@ lfs_migrate() {
                                echo -e "\n$LFS migrate $layout \"$OLDNAME\""
                        fi
                        if $LFS migrate $layout "$OLDNAME"; then
+                               (( $mirror_count > 1 )) &&
+                                       $LFS mirror resync "$OLDNAME"
                                $ECHO "done"
                                # no-op if hlinks empty for 1-link files
                                for link in ${hlinks[*]}; do
@@ -516,7 +523,7 @@ lfs_migrate() {
                fi
 
                # we use --inplace, since we created our own temp file already
-               if ! $RSYNC -a --inplace $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
+               if ! $RSYNC $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
                        echo -e "\r$OLDNAME: copy error, exiting" 1>&2
                        exit 4
                fi
@@ -531,7 +538,8 @@ lfs_migrate() {
                        exit 12
                fi
 
-               $ECHO "done rsync"
+               (( $mirror_count > 1 )) && $LFS mirror resync "$OLDNAME"
+               $ECHO "done $RSYNC"
                # no-op if hlinks empty for 1-link files
                for link in ${hlinks[*]}; do
                        if [ "$link" != "$OLDNAME" ]; then