From: Andreas Dilger <adilger@whamcloud.com>
Date: Thu, 3 Mar 2022 04:51:25 +0000 (-0700)
Subject: LU-15613 utils: add lfs_migrate --cp option
X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=93f31b88e59cf1aecd90f91fd5ecec2c143dbfd0;p=fs%2Flustre-release.git

LU-15613 utils: add lfs_migrate --cp option

Since "lfs_migrate" is only copying a single file at a time, there is
no real benefit from using "rsync" as the only copytool.  Add a short
"--cp" option with suitable arguments to complement "--rsync".

Also allow the copy command to be replaced arbitrarily with RSYNC and
RSYNC_OPTS environment variables to be specified by the caller.

Test-Parameters: trivial
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I66c664948d5cc5a7baa5a65550abfaa8d73ebbe5
Reviewed-on: https://review.whamcloud.com/46691
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
---

diff --git a/lustre/doc/lfs_migrate.1 b/lustre/doc/lfs_migrate.1
index c706ceb..21b09f5 100644
--- a/lustre/doc/lfs_migrate.1
+++ b/lustre/doc/lfs_migrate.1
@@ -5,6 +5,7 @@
 .SH SYNOPSIS
 .B lfs_migrate
 .RB [ "-A " [ -C \fI<cap> \fR] [ -M \fI<min_free> \fR] [ -X \fI<max_free> \fR]]
+.RB [ --cp ]
 .RB [ --dry-run | -n ]
 .RB [ --help | -h ]
 .RB [ --no-rsync | --rsync ]
@@ -70,6 +71,12 @@ directory (potentially changing the stripe count, stripe size, OST pool,
 or OST index of a new file).
 .SH OPTIONS
 .TP
+.B --cp
+Force use of
+.BR cp (1)
+to copy files instead of
+.BR rsync (1).
+.TP
 .B \\--dry-run|-n
 Only print the names of files to be migrated.
 .TP
diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate
index 43ccd60..a480257 100755
--- a/lustre/scripts/lfs_migrate
+++ b/lustre/scripts/lfs_migrate
@@ -12,7 +12,8 @@
 # process, but it would catch the worst cases of in-use files, but
 # to be 100% safe the administrator needs to ensure this is safe.
 
-RSYNC=${RSYNC:-rsync}
+RSYNC=${RSYNC:-"rsync"}
+RSYNC_OPTS=${RSYNC_OPTS:-"-a --inplace"}
 OPT_RSYNC=${LFS_MIGRATE_RSYNC_MODE:-false}
 ECHO=echo
 LFS=${LFS:-lfs}
@@ -45,11 +46,11 @@ old_fid_in_set() {
 
 usage() {
     cat -- <<USAGE 1>&2
-usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
+usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync|--cp]
 		   [--auto-stripe|-A [-C <cap>]
 		   [--min-free|-M <min_free>] [--max-free|-X <max_free>]]
 		   [--pool|-p <pool>] [--stripe-count|-c <stripe_count>]
-		   [--stripe-size|-S <stripe_size>]
+		   [--stripe-size|-S <stripe_size>] [--quiet|-q]
 		   [-D] [-h] [-n] [-S]
 		   [--restripe|-R] [--skip|-s] [--verbose|-v] [--yes|-y] [-0]
 		   [FILE|DIR...]
@@ -59,6 +60,7 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
 		   restripe file using the specified <stripe_count>
 	-C <cap>   when -A is set, limit the migrated file to use on each OST
 		   at most 1/<cap> of the available space of the smallest OST
+	--cp       force the use of 'cp' instead of migrate or rsync
 	-D         do not use direct I/O to copy file contents
 	-h         show this usage message
 	-M <min_free>
@@ -84,8 +86,7 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
 
 Options '-A', '-c', and '-R' are mutually exclusive.
 Options '-C', '-M', and '-X' are ignored if '-A' is not set.
-
-The --rsync and --no-rsync options may not be specified at the same time.
+Options --rsync, --cp, --no-rsync may not be specified at the same time.
 
 If a directory is an argument, all files in the directory are migrated.
 If no file/directory is given, the file list is read from standard input.
@@ -134,32 +135,35 @@ OPT_CAP=100
 while [ -n "$*" ]; do
 	arg="$1"
 	case "$arg" in
+	--auto-stripe|-A) OPT_AUTOSTRIPE=true;;
+	-b|--block|--non-block|--non-direct|-D|--no-verify)
+	   # Always pass non-layout options to 'lfs migrate'
+	   OPT_PASSTHROUGH+=("$arg");;
+	--cp) $OPT_RSYNC && [[ "$RSYNC" != "cp" ]] && OPT_NO_RSYNC=true
+	   OPT_RSYNC=true; RSYNC="cp"; RSYNC_OPTS="-p";;
+	--copy|--yaml|--file) OPT_COMP=true;
+	   # these options have files as arguments, pass both through
+	   OPT_LAYOUT+="$arg $2 "; shift;;
+	--dry-run) OPT_DRYRUN=true; OPT_YES=true;;
+	-c|--stripe-count) OPT_STRIPE_COUNT="$2"; shift;;
+	-C) OPT_CAP="$2"; shift;;
 	-h|--help) usage;;
 	-l|--link) ;; # maintained backward compatibility for now
+	-M|--min-free) OPT_MINFREE="$2"; shift;;
 	-n) OPT_DRYRUN=true; OPT_YES=true
 	   echo "$PROG: -n deprecated, use --dry-run or --non-block" 1>&2;;
-	--dry-run) OPT_DRYRUN=true; OPT_YES=true;;
 	-p|--pool) OPT_POOL="$arg $2"; OPT_LAYOUT+="$OPT_POOL "; shift;;
 	-q|--quiet) ECHO=:;;
 	-R|--restripe) OPT_RESTRIPE=true;;
+	--no-rsync) OPT_NO_RSYNC=true;;
+	--rsync) $OPT_RSYNC && [[ "$RSYNC" != "rsync" ]] && OPT_NO_RSYNC=true
+	   OPT_RSYNC=true ;;
 	-s|--skip) OPT_CHECK=false;;
+	-S|--stripe-size) OPT_STRIPE_SIZE="$2"; shift;;
 	-v|--verbose) OPT_DEBUG=true; ECHO=echo;;
+	-X|--max-free) OPT_MAXFREE="$2"; shift;;
 	-y|--yes) OPT_YES=true;;
 	-0) OPT_NULL=true;;
-	-b|--block|--non-block|--non-direct|-D|--no-verify)
-	   # Always pass non-layout options to 'lfs migrate'
-	   OPT_PASSTHROUGH+=("$arg");;
-	--rsync) OPT_RSYNC=true;;
-	--no-rsync) OPT_NO_RSYNC=true;;
-	--copy|--yaml|--file) OPT_COMP=true;
-	   # these options have files as arguments, pass both through
-	   OPT_LAYOUT+="$arg $2 "; shift;;
-	--auto-stripe|-A) OPT_AUTOSTRIPE=true;;
-	-C) OPT_CAP="$2"; shift;;
-	-M|--min-free) OPT_MINFREE="$2"; shift;;
-	-X|--max-free) OPT_MAXFREE="$2"; shift;;
-	-c|--stripe-count) OPT_STRIPE_COUNT="$2"; shift;;
-	-S|--stripe-size) OPT_STRIPE_SIZE="$2"; shift;;
 	*) # Pass other non-file layout options to 'lfs migrate'
 	   [ -e "$arg" ] && OPT_FILE+="$arg " && break || OPT_LAYOUT+="$arg "
 	esac
@@ -182,7 +186,7 @@ elif $OPT_AUTOSTRIPE && $OPT_RESTRIPE; then
 fi
 
 if $OPT_RSYNC && $OPT_NO_RSYNC; then
-	echo "$PROG: Options --rsync and --no-rsync may not be" \
+	echo "$PROG: Options --cp, --rsync, and --no-rsync may not be" \
 		"specified at the same time." 1>&2
 	exit 1
 fi
@@ -202,8 +206,8 @@ if ! $OPT_YES; then
 fi
 
 # if rsync has --xattr support, then try to copy the xattrs.
-$RSYNC --help 2>&1 | grep -q xattr && RSYNC_OPTS="$RSYNC_OPTS -X"
-$RSYNC --help 2>&1 | grep -q acls && RSYNC_OPTS="$RSYNC_OPTS -A"
+$RSYNC --help 2>&1 | grep -q -- --xattrs && RSYNC_OPTS="$RSYNC_OPTS --xattrs"
+$RSYNC --help 2>&1 | grep -q -- --acls && RSYNC_OPTS="$RSYNC_OPTS --acls"
 # If rsync copies lustre xattrs in the future, then we can skip lfs (bug 22189)
 strings $(which $RSYNC) 2>&1 | grep -q lustre && LFS=:
 
@@ -447,7 +451,8 @@ lfs_migrate() {
 			continue
 		fi
 
-		if ! $OPT_COMP && [ ${layout_info[$l_comp_count]} -gt 0 ]; then
+		if ! $OPT_COMP && ! $OPT_RSYNC &&
+		   (( ${layout_info[$l_comp_count]} > 0 )); then
 			layout+="--copy $OLDNAME"
 			OPT_COMP=true
 		fi
@@ -486,6 +491,8 @@ lfs_migrate() {
 				echo -e "\n$LFS migrate $layout \"$OLDNAME\""
 			fi
 			if $LFS migrate $layout "$OLDNAME"; then
+				(( $mirror_count > 1 )) &&
+					$LFS mirror resync "$OLDNAME"
 				$ECHO "done"
 				# no-op if hlinks empty for 1-link files
 				for link in ${hlinks[*]}; do
@@ -516,7 +523,7 @@ lfs_migrate() {
 		fi
 
 		# we use --inplace, since we created our own temp file already
-		if ! $RSYNC -a --inplace $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
+		if ! $RSYNC $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
 			echo -e "\r$OLDNAME: copy error, exiting" 1>&2
 			exit 4
 		fi
@@ -531,7 +538,8 @@ lfs_migrate() {
 			exit 12
 		fi
 
-		$ECHO "done rsync"
+		(( $mirror_count > 1 )) && $LFS mirror resync "$OLDNAME"
+		$ECHO "done $RSYNC"
 		# no-op if hlinks empty for 1-link files
 		for link in ${hlinks[*]}; do
 			if [ "$link" != "$OLDNAME" ]; then