.RB [ --skip|-s ]
.RB [ --verbose|-v ]
.RB [ --yes|-y ]
+.RB [ -D ]
+.RB [ -A [ -C \fI<cap> \fR] [ -M \fI<min_free> \fR] [ -X \fI<max_free> \fR]]
+.RB [ --stripe-count|-c \fI<stripe_count> \fR]
+.RB [ --stripe-size|-S \fI<stripe_size> \fR]
.RB [ -0 ]
-.RI [ FILE | DIR ]...
+.RI [ FILE | DIR ] ...
.br
.SH DESCRIPTION
.B lfs_migrate
script to indicate a dry-run (no modifications made), and is not passed to
.B lfs migrate
as the non-block option. To specify non-block, use the long option
-.IR --non-block .
+.BR --non-block .
.PP
The current file allocation policies on MDS dictate where the new files
are placed, taking into account whether specific OSTs have been disabled
or OST index of a new file).
.SH OPTIONS
.TP
-.B \\--dry-run
+.B \\--dry-run|-n
Only print the names of files to be migrated.
.TP
+.B \\-D
+Do not use direct I/O to copy file contents.
+.TP
+.B \\-A
+Automatically determine the stripe count for the file, using the algorithm
+count = sqrt(filesize_in_GB) + 1. This option may not be specified at the
+same time as the \fB-c \fRor \\-R \fRoptions.
+.TP
+.B \\--stripe-count|-c \fI<stripe_count>
+Restripe file using the specified \fIstripe_count\fR. This option may not be
+specified at the same time as the \fB-A \fRor \fB-R \fRoptions.
+.TP
+.B \\-C \fI<cap>
+When \fB-A \fRis set, limit the migrated file to use on each OST at most
+1/\fIcap \fRof the available space of the smallest OST. If this option is not
+set, a default value of 100 is used, limiting the object size to 1% of available
+space.
+.TP
.B \\--help|-h
Display help information.
.TP
.B \\--no-rsync
Do not fall back to using rsync if
-.BR lfs (1) " migrate" " fails."
-Cannot be used at the same time as \fI--rsync\fR.
+.BR lfs-migrate (1) " fails."
+Cannot be used at the same time as \fB--rsync\fR.
+.B \\--min-free|-M \fI<min_free>
+When \fB-A \fRis set, only consider OSTs with free space greater than the
+\fImin_free \fRvalue to be available for migration. The value is specified in
+KB. If this option is not set, a default of 256MB is used.
.TP
.B \\--quiet|-q
Run quietly (don't print filenames or status).
.TP
.B \\--rsync
Force rsync to be used instead of
-.BR lfs (1) " migrate" .
-May not be used at the same time as \fI--no-rsync\fR.
+.BR lfs-migrate (1) .
+May not be used at the same time as
+.BR --no-rsync .
.TP
.B \\--restripe|-R
Restripe file using default directory striping instead of keeping striping.
-This option may not be specified at the same time as the -c or -S options
-(these options are passed through to
+This option may not be specified at the same time as the \fB-A\fR, \fB-c\fR, or
+\fB-S \fRoptions. (these options are passed through to
.BR "lfs migrate" ,
and are therefore not listed here).
.TP
.TP
.B \\--verbose|-v
Show verbose debug messages.
+.B \\--max-free|-X \fI<max_free>
+When \fB-A \fRis set, \fImax_free \fRis the maximum amount of free space that
+can be considered available for the migration of the file on each OST. The
+value is specified in KB. This option is useful for testing, by simulating
+OSTs that are nearly full.
+.TP
+.B \\--yes|-y
+Answer 'y' to usage warning without prompting (for scripts; use with caution).
+.SH EXAMPLES
+To rebalance all files within
+.I /testfs/jobs/2011
+.B \\--stripe-size|-S
+.I <stripe_size>
+Restripe file using the specified stripe size. This option may not be
+specified at the same time as the \fB-R \fRoption.
.TP
.B \\--yes|-y
Answer 'y' to usage warning without prompting (for scripts, use with caution).
a guarantee the files are not being modified, that is workload specific) after
disabling file creation on testfs-OST0004 (this is needed on all MDS nodes):
.IP
+.nf
mds# lctl set_param osp.testfs-OST0004*.max_create_count=0
client# lfs find /testfs -obd testfs-OST0004 -size +4G -mtime +2d | lfs_migrate -y
mds# lctl set_param osp.testfs-OST0004*.max_create_count=20000
+.fi
+.PP
+To use automatic striping, and limit the object size per OST to 5% of current
+free space:
+.IP
+lfs_migrate -A -C 20 /testfs/jobs/2011
.SH NOTES
In versions prior to 2.5,
.B lfs_migrate
will be lost.
.SH AVAILABILITY
.B lfs_migrate
-is part of the
-.BR Lustre (7)
+is part of the
+.BR Lustre (7)
filesystem package. Added in the 1.8.4 release.
.SH SEE ALSO
.BR lfs (1)
usage() {
cat -- <<USAGE 1>&2
-usage: lfs_migrate [--dry-run] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
+usage: lfs_migrate [--dry-run|-n] [--help|-h] [--no-rsync|--rsync] [--quiet|-q]
+ [--auto-stripe|-A [-C <cap>]
+ [--min-free|-M <min_free>] [--max-free|-X <max_free>]]
+ [--stripe-count|-c <stripe_count>]
+ [--stripe-size|-S <stripe_size>]
+ [-D] [-h] [-n] [-S]
[--restripe|-R] [--skip|-s] [--verbose|-v] [--yes|-y] [-0]
[FILE|DIR...]
- --dry-run only print the names of files to be migrated
+ -A restripe file using an automatically selected stripe count,
+ uses stripe_count = sqrt(size_in_GB) + 1
+ -c <stripe_count>
+ restripe file using the specified <stripe_count>
+ -C <cap> when -A is set, limit the migrated file to use on each OST
+ at most 1/<cap> of the available space of the smallest OST
+ -D do not use direct I/O to copy file contents
-h show this usage message
+ -M <min_free>
+ when -A is set, an OST must contain more available space than
+ <min_free> KB in order for it to be considered available for
+ use in the migration
--no-rsync do not fall back to rsync mode even if lfs migrate fails
+ -n only print the names of files to be migrated
-q run quietly (don't print filenames or status)
--rsync force rsync mode instead of using lfs migrate
-R restripe file using default directory striping
-s skip file data comparison after migrate
+ -S <stripe_size>
+ restripe file using the specified stripe size
-v show verbose debug messages
+ -X <max_free>
+ when -A is set, limit the amount of space on each OST that
+ can be considered available for the migration to
+ <max_free> KB
-y answer 'y' to usage question
-0 input file names on stdin are separated by a null character
-If the --restripe|-R option is used, other "lfs setstripe" layout options
-such as -E, -c, -S, --copy, and --yaml may not be specified at the same time.
-Only the --block, --non-block, --non-direct, and --verbose non-layout setstripe
-options may be used in that case.
+Options '-A', '-c', and '-R' are mutually exclusive.
+Options '-C', '-M', and '-X' are ignored if '-A' is not set.
The --rsync and --no-rsync options may not be specified at the same time.
OPT_PASSTHROUGH=()
OPT_RESTRIPE=false
OPT_YES=false
+LFS_OPT_DIRECTIO=""
+OPT_AUTOSTRIPE=false
+OPT_STRIPE_COUNT=""
+OPT_STRIPE_SIZE=""
+OPT_MINFREE=262144
+OPT_MAXFREE=""
+OPT_CAP=100
# Examine any long options and arguments. getopts does not support long
# options, so they must be stripped out and classified as either options
--copy|--yaml|--file)
# these options have files as arguments, pass both through
OPT_LAYOUT+="$arg $2"; shift;;
+ --auto-stripe|-A) OPT_AUTOSTRIPE=true;;
+ -C) OPT_CAP="$2"; shift;;
+ -D) LFS_OPT_DIRECTIO="-D";;
+ -M|--min-free) OPT_MINFREE="$2"; shift;;
+ -X|--max-free) OPT_MAXFREE="$2"; shift;;
+ -c|--stripe-count) OPT_STRIPE_COUNT="$2"; shift;;
+ -S|--stripe-size) OPT_STRIPE_SIZE="$2"; shift;;
*) # Pass other non-file layout options to 'lfs migrate'
[ -e "$arg" ] && OPT_FILE+="$arg " && break || OPT_LAYOUT+="$arg "
esac
if $OPT_RESTRIPE && [ -n "$OPT_LAYOUT" ]; then
echo "$PROG: Options $OPT_LAYOUT cannot be used with the -R option" 1>&2
exit 1
+elif $OPT_RESTRIPE && [[ "$OPT_STRIPE_COUNT" || "$OPT_STRIPE_SIZE" ]]; then
+ echo "$(basename $0): Options -c <stripe_count> and -S <stripe_size> "\
+ "may not be specified at the same time as the -R option." 1>&2
+ exit 1
+elif $OPT_AUTOSTRIPE && [ -n "$OPT_STRIPE_COUNT" ]; then
+ echo ""
+ echo "$(basename $0) error: The -c <stripe_count> option may not" 1>&2
+ echo "be specified at the same time as the -A option." 1>&2
+ exit 1
+elif $OPT_AUTOSTRIPE && $OPT_RESTRIPE; then
+ echo ""
+ echo "$(basename $0) error: The -A option may not be specified at" 1>&2
+ echo "the same time as the -R option." 1>&2
+ exit 1
fi
if $OPT_RSYNC && $OPT_NO_RSYNC; then
# access the temporary file.
umask 0077
+# Use stripe count = sqrt(size_in_GB) + 1, but cap object size per OST.
+function calc_stripe()
+{
+ local filename=$1
+ local filekb=$2
+ local obj_max_kb=$3
+ local filegb=$((filekb / 1048576))
+ local stripe_count=1
+ local ost_max_count=0
+
+ # Files up to 1GB will have 1 stripe if they fit within the object max
+ if [[ $filegb -lt 1 && "$obj_max_kb" && $filekb -le $obj_max_kb ]]; then
+ echo 1 "$obj_max_kb" && return
+ fi
+
+ stripe_count=$(bc <<< "scale=0; 1 + sqrt($filegb)" 2> /dev/null) ||
+ { echo "cannot auto calculate stripe count" >&2; return; }
+
+ if [ -z "$obj_max_kb" ]; then
+ local ost_min_kb=$((1 << 62))
+
+ # Calculate cap on object size at 1% of smallest OST
+ # but only include OSTs that have 256MB+ available space
+ while IFS='' read avail; do
+ [[ "$OPT_MAXFREE" && $avail -gt $OPT_MAXFREE ]] &&
+ avail=$OPT_MAXFREE
+ if [ $avail -ge $OPT_MINFREE ]; then
+ ost_max_count=$((ost_max_count + 1))
+ if [ $avail -lt $ost_min_kb ]; then
+ ost_min_kb=$avail
+ fi
+ fi
+ done < <($LFS df $OLDNAME | awk '/OST/ { print $4 }')
+ # Once this script supports pools, the lfs df command above
+ # should also include the -p <pool> option to restrict the
+ # listed OSTs to the correct pool.
+
+ if [ $ost_max_count -eq 0 ]; then
+ echo "no OSTs with sufficient available space" >&2
+ return
+ fi
+
+ if [ "$ost_min_kb" -eq $((1 << 62)) ]; then
+ echo "warning: unable to determine minimum OST size, " \
+ "object size not capped" >&2
+ obj_max_kb=0
+ echo "$stripe_count" "$obj_max_kb"
+ return
+ fi
+
+ obj_max_kb=$((ost_min_kb / $OPT_CAP))
+ elif [ $obj_max_kb -eq 0 ]; then
+ echo "warning: unable to determine minimum OST size " \
+ "from previous migrate, object size not capped" >&2
+ echo "$stripe_count" "$obj_max_kb"
+ return
+ fi
+
+ # If disk usage would exceed the cap, increase the number of stripes
+ [ $filekb -gt $((stripe_count * $obj_max_kb)) ] &&
+ stripe_count=$((filekb / $obj_max_kb))
+
+ # Limit the count to the number of eligible OSTs
+ if [ "$stripe_count" -gt $ost_max_count ]; then
+ echo "$ost_max_count" "$obj_max_kb"
+ else
+ echo "$stripe_count" "$obj_max_kb"
+ fi
+}
+
lfs_migrate() {
while IFS='' read -d '' OLDNAME; do
local hlinks=()
- local stripe_size
- local stripe_count
+ local stripe_size="$OPT_STRIPE_SIZE"
+ local stripe_count="$OPT_STRIPE_COUNT"
+ local parent_count=""
+ local parent_size=""
local stripe_pool
local mirror_count
local layout
$ECHO -n "$OLDNAME: "
# avoid duplicate stat if possible
- local nlink_type=($(LANG=C stat -c "%h %F" "$OLDNAME" \
+ local nlink_type=($(LANG=C stat -c "%h %F %s" "$OLDNAME" \
2> /dev/null))
# skip non-regular files, since they don't have any objects
# and there is no point in trying to migrate them.
if [ "${nlink_type[1]}" != "regular" ]; then
- echo -e "\r\e[K$OLDNAME: not a regular file, skipped"
+ echo -e "$OLDNAME: not a regular file, skipped" 1>&2
continue
fi
# working out write perms is hard, let the shell do it
if [ ! -w "$OLDNAME" ]; then
- echo -e "\r\e[K$OLDNAME: no write permission, skipped"
+ echo -e "$OLDNAME: no write permission, skipped" 1>&2
continue
fi
# also absolute so that the names can be compared
local oldname_absolute=$(readlink -f "$OLDNAME")
if [ -z "$oldname_absolute" ]; then
- echo -e "\r\e[K$OLDNAME: cannot resolve full path, skipped"
+ echo -e "$OLDNAME: cannot resolve full path, skipped" 1>&2
continue
fi
OLDNAME=$oldname_absolute
if [ $? -ne 0 ]; then
echo -n "\r\e[K$OLDNAME: cannot determine FID; skipping; "
echo "is this a Lustre file system?"
+ echo -e "$OLDNAME: cannot determine FID; skipping; " 1>&2
+ echo "is this a Lustre file system?" 1>&2
continue
fi
if [[ ${nlink_type[0]} -gt 1 ]] || $RSYNC_WITH_HLINKS; then
# don't migrate a hard link if it was already migrated
if path_in_set "$OLDNAME"; then
- $ECHO -e "$OLDNAME: already migrated via another hard link"
+ $ECHO "\r\e[Kalready migrated via another hard link"
continue
fi
if $OPT_RESTRIPE; then
UNLINK=""
else
- # if rsync copies Lustre xattrs properly in the future
- # (i.e. before the file data, so that it preserves striping)
- # then we don't need to do this getstripe/mktemp stuff.
+ # if rsync copies Lustre xattrs properly in the future
+ # (i.e. before the file data, so that it preserves
+ # striping) then we don't need to do this getstripe
+ # stuff.
UNLINK="-u"
- stripe_count=$($LFS getstripe -c "$OLDNAME" 2> /dev/null)
- stripe_size=$($LFS getstripe -S "$OLDNAME" 2> /dev/null)
stripe_pool=$($LFS getstripe -p "$OLDNAME" 2> /dev/null)
mirror_count=$($LFS getstripe -N "$OLDFILE" 2> /dev/null)
+ if $OPT_AUTOSTRIPE; then
+ local filekb=$((${nlink_type[3]} / 1024))
+ read stripe_count OBJ_MAX_KB < <(calc_stripe \
+ "$OLDNAME" "$filekb" "$OBJ_MAX_KB")
+ [ -z "$stripe_count" ] && exit 1
+ [ $stripe_count -lt 1 ] && stripe_count=1
+ else
+ [ "$OPT_STRIPE_COUNT" ] && stripe_count=$OPT_STRIPE_COUNT ||
+ stripe_count=$($LFS getstripe -c "$OLDNAME" \
+ 2> /dev/null)
+ fi
+ [ -z "$stripe_size" ] &&
+ stripe_size=$($LFS getstripe -S "$OLDNAME" 2> /dev/null)
+
[ -z "$stripe_count" -o -z "$stripe_size" ] && UNLINK=""
fi
continue
fi
- stripe_count="-c $stripe_count"
- stripe_size="-S $stripe_size"
+ [ -n "$stripe_count" ] && stripe_count="-c $stripe_count"
+ [ -n "$stripe_size" ] && stripe_size="-S $stripe_size"
[ -n "$stripe_pool" ] && stripe_pool="-p $stripe_pool"
[ -n "$mirror_count" ] && mirror_count="-N $mirror_count"
layout="$stripe_count $stripe_size $stripe_pool $mirror_count \
local mntpoint=$(df -P "$OLDNAME" |
awk 'NR==2 { print $NF; exit }')
if [ -z "$mntpoint" ]; then
- echo -e "\r\e[K$OLDNAME: cannot determine mount point; skipped"
+ echo -e "$OLDNAME: cannot determine mount point; skipped" 1>&2
continue
fi
hlinks=$($LFS fid2path "$mntpoint" "$fid" 2> /dev/null)
if [ $? -ne 0 ]; then
- echo -e "\r\e[K$OLDNAME: cannot determine hard link paths, skipped"
+ echo -e "$OLDNAME: cannot determine hard link paths, skipped" 1>&2
continue
fi
hlinks+=("$OLDNAME")
done
continue
elif $OPT_NO_RSYNC; then
- echo -e "\r\e[K$OLDNAME: refusing to fall back to rsync, skipped" 1>&2
+ echo -e "$OLDNAME: refusing to fall back to rsync, skipped" 1>&2
continue
else
$ECHO -n "falling back to rsync: "
NEWNAME=$(mktemp $UNLINK "$OLDNAME-lfs_migrate.tmp.XXXXXX")
if [ $? -ne 0 -o -z "$NEWNAME" ]; then
- echo -e "\r\e[K$OLDNAME: can't make temp file, skipped" 1>&2
+ echo -e "$OLDNAME: cannot make temp file, skipped" 1>&2
continue
fi
if [ "$UNLINK" ]; then
- if ! $LFS setstripe "${OPT_PASSTHROUGH}" $layout \
+ if ! $LFS setstripe "${OPT_PASSTHROUGH[@]}" $layout \
"$NEWNAME"; then
echo -e "\r\e[K$NEWNAME: setstripe failed, exiting" 1>&2
exit 2
# we use --inplace, since we created our own temp file already
if ! $RSYNC -a --inplace $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
- echo -e "\r\e[K$OLDNAME: copy error, exiting" 1>&2
+ echo -e "$OLDNAME: copy error, exiting" 1>&2
exit 4
fi
if $OPT_CHECK && ! cmp -s "$OLDNAME" "$NEWNAME"; then
- echo -e "\r\e[K$NEWNAME: compare failed, exiting" 1>&2
+ echo -e "$NEWNAME: compare failed, exiting" 1>&2
exit 8
fi
if ! mv "$NEWNAME" "$OLDNAME"; then
- echo -e "\r\e[K$OLDNAME: rename error, exiting" 1>&2
+ echo -e "$OLDNAME: rename error, exiting" 1>&2
exit 12
fi
test_mkdir "$dir" || error "cannot create dir $dir"
echo "testing lfs migrate mode when all links fit within xattrs"
- LFS_MIGRATE_RSYNC=false check_migrate_links "$dir" 2 99
+ LFS_MIGRATE_RSYNC_MODE=false check_migrate_links "$dir" 2 99
echo "testing rsync mode when all links fit within xattrs"
- LFS_MIGRATE_RSYNC=true check_migrate_links "$dir" 2 99
+ LFS_MIGRATE_RSYNC_MODE=true check_migrate_links "$dir" 2 99
echo "testing lfs migrate mode when all links do not fit within xattrs"
- LFS_MIGRATE_RSYNC=false check_migrate_links "$dir" 101 100
+ LFS_MIGRATE_RSYNC_MODE=false check_migrate_links "$dir" 101 100
echo "testing rsync mode when all links do not fit within xattrs"
- LFS_MIGRATE_RSYNC=true check_migrate_links "$dir" 101 100
+ LFS_MIGRATE_RSYNC_MODE=true check_migrate_links "$dir" 101 100
+
# clean up
rm -rf $dir
}
run_test 56xb "lfs migration hard link support"
+test_56xc() {
+ [[ $OSTCOUNT -lt 2 ]] && skip_env "needs >= 2 OSTs"
+
+ local dir="$DIR/$tdir"
+
+ test_mkdir "$dir" || error "cannot create dir $dir"
+
+ # Test 1: ensure file < 1 GB is always migrated with 1 stripe
+ echo -n "Setting initial stripe for 20MB test file..."
+ $LFS setstripe -c 2 -i 0 "$dir/20mb" || error "cannot setstripe"
+ echo "done"
+ echo -n "Sizing 20MB test file..."
+ truncate "$dir/20mb" 20971520 || error "cannot create 20MB test file"
+ echo "done"
+ echo -n "Verifying small file autostripe count is 1..."
+ $LFS_MIGRATE -y -A -C 1 "$dir/20mb" &> /dev/null ||
+ error "cannot migrate 20MB file"
+ local stripe_count=$($LFS getstripe -c "$dir/20mb") ||
+ error "cannot get stripe for $dir/20mb"
+ [ $stripe_count -eq 1 ] ||
+ error "unexpected stripe count $stripe_count for 20MB file"
+ rm -f "$dir/20mb"
+ echo "done"
+
+ # Test 2: File is small enough to fit within the available space on
+ # sqrt(size_in_gb) + 1 OSTs but is larger than 1GB. The file must
+ # have at least an additional 1KB for each desired stripe for test 3
+ echo -n "Setting stripe for 1GB test file..."
+ $LFS setstripe -c 1 -i 0 "$dir/1gb" || error "cannot setstripe"
+ echo "done"
+ echo -n "Sizing 1GB test file..."
+ # File size is 1GB + 3KB
+ truncate "$dir/1gb" 1073744896 &> /dev/null ||
+ error "cannot create 1GB test file"
+ echo "done"
+ echo -n "Migrating 1GB file..."
+ $LFS_MIGRATE -y -A -C 1 "$dir/1gb" &> /dev/null ||
+ error "cannot migrate file"
+ echo "done"
+ echo -n "Verifying autostripe count is sqrt(n) + 1..."
+ stripe_count=$($LFS getstripe -c "$dir/1gb") ||
+ error "cannot get stripe for $dir/1gb"
+ [ $stripe_count -eq 2 ] ||
+ error "unexpected stripe count $stripe_count (expected 2)"
+ echo "done"
+
+ # Test 3: File is too large to fit within the available space on
+ # sqrt(n) + 1 OSTs. Simulate limited available space with -X
+ if [ $OSTCOUNT -ge 3 ]; then
+ # The required available space is calculated as
+ # file size (1GB + 3KB) / OST count (3).
+ local kb_per_ost=349526
+
+ echo -n "Migrating 1GB file..."
+ $LFS_MIGRATE -y -A -C 1 -X $kb_per_ost "$dir/1gb" &>> \
+ /dev/null || error "cannot migrate file"
+ echo "done"
+
+ stripe_count=$($LFS getstripe -c "$dir/1gb")
+ echo -n "Verifying autostripe count with limited space..."
+ [ "$stripe_count" -a $stripe_count -eq 3 ] ||
+ error "unexpected stripe count $stripe_count (wanted 3)"
+ echo "done"
+ fi
+
+ # clean up
+ rm -rf $dir
+}
+run_test 56xc "lfs migration autostripe"
+
test_56y() {
[ $MDS1_VERSION -lt $(version_code 2.4.53) ] &&
skip "No HSM $(lustre_build_version $SINGLEMDS) MDS < 2.4.53"
git://git.whamcloud.com / fs / lustre-release.git / commitdiff