From: Andreas Dilger <adilger@whamcloud.com>
Date: Thu, 3 Mar 2022 04:51:25 +0000 (-0700)
Subject: LU-15613 utils: add lfs_migrate --cp option
X-Git-Tag: 2.15.91~25
X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=37b8114c5ff92d7447321c1513515503f442b6bd;p=fs%2Flustre-release.git

LU-15613 utils: add lfs_migrate --cp option

Since "lfs_migrate" is only copying a single file at a time, there is
no real benefit from using "rsync" as the only copytool.  Add a short
"--cp" option with suitable arguments to complement "--rsync".

Also allow the copy command to be replaced arbitrarily with RSYNC and
RSYNC_OPTS environment variables to be specified by the caller.

Test-Parameters: trivial
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I66c664948d5cc5a7baa5a65550abfaa8d73ebbe5
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49727
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Feng Lei <flei@whamcloud.com>
Reviewed-by: Jian Yu <yujian@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---

diff --git a/lustre/doc/lfs_migrate.1 b/lustre/doc/lfs_migrate.1
index fc9ffaba..63b25ff 100644
--- a/lustre/doc/lfs_migrate.1
+++ b/lustre/doc/lfs_migrate.1
@@ -5,6 +5,7 @@
 .SH SYNOPSIS
 .B lfs_migrate
 .RB [ "-A " [ -C \fI<cap> \fR] [ -M \fI<min_free> \fR] [ -X \fI<max_free> \fR]]
+.RB [ --cp ]
 .RB [ --dry-run | -n ]
 .RB [ --help | -h ]
 .RB [ --no-rsync | --rsync ]
@@ -70,6 +71,12 @@ directory (potentially changing the stripe count, stripe size, OST pool,
 or OST index of a new file).
 .SH OPTIONS
 .TP
+.B --cp
+Force use of
+.BR cp (1)
+to copy files instead of
+.BR rsync (1).
+.TP
 .B \\--dry-run|-n
 Only print the names of files to be migrated.
 .TP
diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate
index c81175f..dbc5ad2 100755
--- a/lustre/scripts/lfs_migrate
+++ b/lustre/scripts/lfs_migrate
@@ -12,7 +12,8 @@
 # process, but it would catch the worst cases of in-use files, but
 # to be 100% safe the administrator needs to ensure this is safe.
 
-RSYNC=${RSYNC:-rsync}
+RSYNC=${RSYNC:-"rsync"}
+RSYNC_OPTS=${RSYNC_OPTS:-"-a --inplace"}
 ECHO=echo
 LFS=${LFS:-lfs}
 RSYNC_WITH_HLINKS=false
@@ -44,11 +45,11 @@ old_fid_in_set() {
 
 usage() {
     cat -- <<USAGE 1>&2
-usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
+usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync|--cp]
 		   [--auto-stripe|-A [-C <cap>]
 		   [--min-free|-M <min_free>] [--max-free|-X <max_free>]]
 		   [--pool|-p <pool>] [--stripe-count|-c <stripe_count>]
-		   [--stripe-size|-S <stripe_size>]
+		   [--stripe-size|-S <stripe_size>] [--quiet|-q]
 		   [-D] [-h] [-n] [-S]
 		   [--restripe|-R] [--skip|-s] [--verbose|-v] [--yes|-y] [-0]
 		   [FILE|DIR...]
@@ -58,6 +59,7 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
 		   restripe file using the specified <stripe_count>
 	-C <cap>   when -A is set, limit the migrated file to use on each OST
 		   at most 1/<cap> of the available space of the smallest OST
+	--cp       force the use of 'cp' instead of migrate or rsync
 	-D         do not use direct I/O to copy file contents
 	-h         show this usage message
 	-M <min_free>
@@ -78,12 +80,13 @@ usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
 		   when -A is set, limit the amount of space on each OST that
 		   can be considered available for the migration to
 		   <max_free> KB
-	-y         answer 'y' to usage question (only when --rsync used)
+	-y         answer 'y' to usage question (only when --rsync/--cp used)
 	-0         input file names on stdin are separated by a null character
 
 Options '-A', and '-R' are mutually exclusive with each other, and any
 specific layout (e.g. any specific parameters like '-c', '-S', '-E', '-p').
 Options '-C', '-M', and '-X' are ignored if '-A' is not set.
+Options --rsync, --cp, --no-rsync may not be specified at the same time.
 
 If a directory is an argument, all files in the directory are migrated.
 If no file/directory is given, the file list is read from standard input.
@@ -139,40 +142,43 @@ OPT_CAP=100
 while [ -n "$*" ]; do
 	arg="$1"
 	case "$arg" in
-	-h|--help) usage;;
-	-l|--link) ;; # maintained backward compatibility for now
-	-n) OPT_DRYRUN=true; OPT_YES=true
-	   echo "$PROG: -n deprecated, use --dry-run or --non-block" 1>&2;;
-	--dry-run) OPT_DRYRUN=true; OPT_YES=true;;
-	-p|--pool) # if within a component, pass through pool
-		$OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_POOL="-p $2";
-		shift;;
-	-q|--quiet) ECHO=:;;
-	-R|--restripe) OPT_RESTRIPE=true;;
-	-s|--skip) OPT_CHECK=false;;
-	-v|--verbose) OPT_DEBUG=true; ECHO=echo;;
-	-y|--yes) OPT_YES=true;;
-	-0) OPT_NULL=true;;
+	--auto-stripe|-A) OPT_AUTOSTRIPE=true;;
 	-b|--block|--non-block|--non-direct|-D|--no-verify)
 	   # Always pass non-layout options to 'lfs migrate'
 	   OPT_PASSTHROUGH+=("$arg");;
-	--rsync) OPT_RSYNC=true; OPT_NO_RSYNC=false;;
-	--no-rsync) OPT_NO_RSYNC=true; OPT_RSYNC=false;;
+	--cp) $OPT_RSYNC && [[ "$RSYNC" != "cp" ]] && OPT_NO_RSYNC=true
+	   OPT_RSYNC=true; RSYNC="cp"; RSYNC_OPTS="-p";;
 	--copy|--yaml|--file) OPT_COPY=true;
 	   # these options have files as arguments, pass both through
 	   OPT_LAYOUT+="$arg $2 "; shift;;
-	--auto-stripe|-A) OPT_AUTOSTRIPE=true;;
+	--dry-run) OPT_DRYRUN=true; OPT_YES=true;;
+	-c|--stripe-count) # if within a component, pass through stripe_count
+		$OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_STRIPE_COUNT="$2"
+		shift;;
 	-C) OPT_CAP="$2"; shift;;
 	-D) LFS_OPT_DIRECTIO="-D";;
 	-E|--comp-end|--component-end) OPT_COMP=true; OPT_LAYOUT+="$arg ";;
+	-h|--help) usage;;
+	-l|--link) ;; # maintained backward compatibility for now
 	-M|--min-free) OPT_MINFREE="$2"; shift;;
-	-X|--max-free) OPT_MAXFREE="$2"; shift;;
-	-c|--stripe-count) # if within a component, pass through stripe_count
-		$OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_STRIPE_COUNT="$2"
+	-n) OPT_DRYRUN=true; OPT_YES=true
+	   echo "$PROG: -n deprecated, use --dry-run or --non-block" 1>&2;;
+	-p|--pool) # if within a component, pass through pool
+		$OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_POOL="-p $2";
 		shift;;
+	-q|--quiet) ECHO=:;;
+	-R|--restripe) OPT_RESTRIPE=true;;
+	--no-rsync) OPT_NO_RSYNC=true; OPT_RSYNC=false;;
+	--rsync) [[ "$RSYNC" != "rsync" ]] && OPT_NO_RSYNC=true || OPT_NO_RSYNC=false
+	   OPT_RSYNC=true ;;
+	-s|--skip) OPT_CHECK=false;;
 	-S|--stripe-size) # if within a component, pass through stripe_size
 		$OPT_COMP && OPT_LAYOUT+="$arg $2 " || OPT_STRIPE_SIZE="$2"
 		shift;;
+	-v|--verbose) OPT_DEBUG=true; ECHO=echo;;
+	-X|--max-free) OPT_MAXFREE="$2"; shift;;
+	-y|--yes) OPT_YES=true;;
+	-0) OPT_NULL=true;;
 	*) # Pass other non-file layout options to 'lfs migrate'
 	   [[ -e "$arg" ]] && OPT_FILE+="$arg " && break || OPT_LAYOUT+="$arg "
 	esac
@@ -191,6 +197,12 @@ elif $OPT_RESTRIPE && $OPT_AUTOSTRIPE; then
 	exit 1
 fi
 
+if $OPT_RSYNC && $OPT_NO_RSYNC; then
+	echo "$PROG: Options --cp, --rsync, and --no-rsync may not be" \
+		"specified at the same time." 1>&2
+	exit 1
+fi
+
 if $OPT_RSYNC && ! $OPT_YES; then
 	echo ""
 	echo "'lfs_migrate --rsync' is NOT SAFE for moving in-use files." 1>&2
@@ -202,8 +214,8 @@ if $OPT_RSYNC && ! $OPT_YES; then
 fi
 
 # if rsync has --xattr support, then try to copy the xattrs.
-$RSYNC --help 2>&1 | grep -q xattr && RSYNC_OPTS="$RSYNC_OPTS -X"
-$RSYNC --help 2>&1 | grep -q acls && RSYNC_OPTS="$RSYNC_OPTS -A"
+$RSYNC --help 2>&1 | grep -q -- --xattrs && RSYNC_OPTS="$RSYNC_OPTS --xattrs"
+$RSYNC --help 2>&1 | grep -q -- --acls && RSYNC_OPTS="$RSYNC_OPTS --acls"
 # If rsync copies lustre xattrs in the future, then we can skip lfs (bug 22189)
 strings $(which $RSYNC) 2>&1 | grep -q lustre && LFS=:
 
@@ -461,8 +473,8 @@ lfs_migrate() {
 			continue
 		fi
 
-		if ! $OPT_COPY && ! $OPT_COMP &&
-		   [[ ${layout_info[$l_comp_count]} > 0 ]]; then
+		if ! $OPT_COPY && ! $OPT_COMP && ! $OPT_RSYNC &&
+		   (( ${layout_info[$l_comp_count]} > 0 )); then
 			layout+="--copy $OLDNAME"
 			OPT_COPY=true
 		fi
@@ -501,6 +513,8 @@ lfs_migrate() {
 				echo -e "\n$LFS migrate $layout \"$OLDNAME\""
 			fi
 			if $LFS migrate $layout "$OLDNAME"; then
+				(( $mirror_count > 1 )) &&
+					$LFS mirror resync "$OLDNAME"
 				$ECHO "done"
 				# no-op if hlinks empty for 1-link files
 				for link in ${hlinks[*]}; do
@@ -531,7 +545,7 @@ lfs_migrate() {
 		fi
 
 		# we use --inplace, since we created our own temp file already
-		if ! $RSYNC -a --inplace $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
+		if ! $RSYNC $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
 			echo -e "\r$OLDNAME: copy error, exiting" 1>&2
 			exit 4
 		fi
@@ -546,7 +560,8 @@ lfs_migrate() {
 			exit 12
 		fi
 
-		$ECHO "done rsync"
+		(( $mirror_count > 1 )) && $LFS mirror resync "$OLDNAME"
+		$ECHO "done $RSYNC"
 		# no-op if hlinks empty for 1-link files
 		for link in ${hlinks[*]}; do
 			if [ "$link" != "$OLDNAME" ]; then