LU-12157 utils: fix lfs_migrate output and testing

Don't pass the "-v" option through to "lfs migrate", as this causes
the filename to be printed twice when run with the "-v" option.

Don't use "echo -e" to process escape characters in filenames unless
this is needed, but add it where it is needed.  Don't add ANSI escape
characters to the output.  The output previously looked like:

     /mnt/testfs/l0: stripe count=1,size=1048576,pool=/mnt/testfs/l0
     done migrate
     nr[K/mnt/testfs/l1: /mnt/testfs/l1: already migrated ...
     nr[K/mnt/testfs/l2: /mnt/testfs/l1: already migrated ...
     nr[K/mnt/testfs/l3: /mnt/testfs/l1: already migrated ...

Print out the "pool=" and "mirror_count=" parameters only if needed.

Fix "-A" option to round up the number of stripes when the migrated.
Skip sanity test_56xc 1GB test if there is not enough space on OSTs.

Fixes: 60c5bc2502 ("LU-8235 scripts: pass unrecognized options to lfs migrate")
Fixes: 80a2ff7137 ("LU-6051 utils: allow lfs_migrate to handle hard links")
Fixes: 99d7a8ed43 ("LU-8207 scripts: add auto-stripe option to lfs_migrate")
Test-Parameters: trivial fstype=zfs testlist=sanity envdefinitions=ONLY=56
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I059e7daeb2fa82e7607fd9d862797433053ebbe5
Reviewed-on: https://review.whamcloud.com/34592
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Ben Evans <bevans@cray.com>
Reviewed-by: James Nunez <jnunez@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/scripts/lfs_migrate b/lustre/scripts/lfs_migrate
index 0fff6d1..eb0799d 100755 (executable)
--- a/lustre/scripts/lfs_migrate
+++ b/lustre/scripts/lfs_migrate
@@ -27,7 +27,7 @@ add_to_set() {
        local old_fid="$1"
        local path="$2"
 
-       echo -e "$old_fid $path" >> "$MIGRATED_SET"
+       echo "$old_fid $path" >> "$MIGRATED_SET"
 }
 
 path_in_set() {
@@ -139,7 +139,7 @@ while [ -n "$*" ]; do
        -q|--quiet) ECHO=:;;
        -R|--restripe) OPT_RESTRIPE=true;;
        -s|--skip) OPT_CHECK=false;;
-       -v|--verbose) OPT_DEBUG=true; ECHO=echo; OPT_PASSTHROUGH+=("$arg");;
+       -v|--verbose) OPT_DEBUG=true; ECHO=echo;;
        -y|--yes) OPT_YES=true;;
        -0) OPT_NULL=true;;
        -b|--block|--non-block|--non-direct|--no-verify)
@@ -256,7 +256,7 @@ function calc_stripe()
                        return
                fi
 
-               if [ "$ost_min_kb" -eq $((1 << 62)) ]; then
+               if (( ost_min_kb == (1 << 62) )); then
                        echo "warning: unable to determine minimum OST size, " \
                             "object size not capped" >&2
                        obj_max_kb=0
@@ -272,9 +272,10 @@ function calc_stripe()
                return
        fi
 
-       # If disk usage would exceed the cap, increase the number of stripes
-       [ $filekb -gt $((stripe_count * $obj_max_kb)) ] &&
-               stripe_count=$((filekb / $obj_max_kb))
+       # If disk usage would exceed the cap, increase the number of stripes.
+       # Round up to the nearest MB to ensure file will fit.
+       (( filekb > stripe_count * obj_max_kb )) &&
+               stripe_count=$(((filekb + obj_max_kb - 1024) / obj_max_kb))
 
        # Limit the count to the number of eligible OSTs
        if [ "$stripe_count" -gt $ost_max_count ]; then
@@ -304,13 +305,13 @@ lfs_migrate() {
                # skip non-regular files, since they don't have any objects
                # and there is no point in trying to migrate them.
                if [ "${nlink_type[1]}" != "regular" ]; then
-                       echo -e "$OLDNAME: not a regular file, skipped" 1>&2
+                       echo -e "\r$OLDNAME: not a regular file, skipped" 1>&2
                        continue
                fi
 
                # working out write perms is hard, let the shell do it
                if [ ! -w "$OLDNAME" ]; then
-                       echo -e "$OLDNAME: no write permission, skipped" 1>&2
+                       echo -e "\r$OLDNAME: no write permission, skipped" 1>&2
                        continue
                fi
 
@@ -323,7 +324,7 @@ lfs_migrate() {
                # also absolute so that the names can be compared
                local oldname_absolute=$(readlink -f "$OLDNAME")
                if [ -z "$oldname_absolute" ]; then
-                       echo -e "$OLDNAME: cannot resolve full path, skipped" 1>&2
+                       echo -e "\r$OLDNAME: cannot resolve full path, skipped" 1>&2
                        continue
                fi
                OLDNAME=$oldname_absolute
@@ -334,17 +335,15 @@ lfs_migrate() {
                # for detecting unlisted hard links could then be removed.
                local fid=$($LFS path2fid "$OLDNAME" 2> /dev/null)
                if [ $? -ne 0 ]; then
-                       echo -n "\r\e[K$OLDNAME: cannot determine FID; skipping; "
-                       echo "is this a Lustre file system?"
-                       echo -e "$OLDNAME: cannot determine FID; skipping; " 1>&2
-                       echo "is this a Lustre file system?" 1>&2
+                       echo -e "\r$OLDNAME: cannot determine FID; skipping; " \
+                               "is this a Lustre file system?" 1>&2
                        continue
                fi
 
                if [[ ${nlink_type[0]} -gt 1 ]] || $RSYNC_WITH_HLINKS; then
                        # don't migrate a hard link if it was already migrated
                        if path_in_set "$OLDNAME"; then
-                               $ECHO "\r\e[Kalready migrated via another hard link"
+                               $ECHO "already migrated via another hard link"
                                continue
                        fi
 
@@ -355,7 +354,7 @@ lfs_migrate() {
                        # being migrated.
                        local migrated=$(old_fid_in_set "$fid")
                        if [ -n "$migrated" ]; then
-                               $ECHO -e "$OLDNAME: already migrated via another hard link"
+                               $ECHO "already migrated via another hard link"
                                if $OPT_RSYNC; then
                                        # Only the rsync case has to relink.
                                        # The lfs migrate case preserves the
@@ -383,6 +382,7 @@ lfs_migrate() {
 
                        if $OPT_AUTOSTRIPE; then
                                local filekb=$((${nlink_type[3]} / 1024))
+
                                read stripe_count OBJ_MAX_KB < <(calc_stripe \
                                        "$OLDNAME" "$filekb" "$OBJ_MAX_KB")
                                [ -z "$stripe_count" ] && exit 1
@@ -417,15 +417,16 @@ lfs_migrate() {
                                               /dev/null)
                        fi
 
-                       $ECHO -n "stripe" \
-                               "count=${stripe_count:-$parent_count}," \
-                               "size=${stripe_size:-$parent_size}," \
-                               "pool=${stripe_pool}," \
-                               "mirror_count=${mirror_count}"
+                       $ECHO -n "stripe_count=${stripe_count:-$parent_count},stripe_size=${stripe_size:-$parent_size}"
+                       [ -n "$stripe_pool" ] &&
+                               $ECHO -n ",pool=${stripe_pool}"
+                       [ -n "$mirror_count" ] &&
+                               $ECHO -n ",mirror_count=${mirror_count}"
+                       $ECHO -n " "
                fi
 
                if $OPT_DRYRUN; then
-                       $ECHO "dry run, skipped"
+                       $ECHO " dry run, skipped"
                        continue
                fi
 
@@ -441,12 +442,12 @@ lfs_migrate() {
                local mntpoint=$(df -P "$OLDNAME" |
                                awk 'NR==2 { print $NF; exit }')
                if [ -z "$mntpoint" ]; then
-                       echo -e "$OLDNAME: cannot determine mount point; skipped" 1>&2
+                       echo -e "\r$OLDNAME: cannot determine mount point; skipped" 1>&2
                        continue
                fi
                hlinks=$($LFS fid2path "$mntpoint" "$fid" 2> /dev/null)
                if [ $? -ne 0 ]; then
-                       echo -e "$OLDNAME: cannot determine hard link paths, skipped" 1>&2
+                       echo -e "\r$OLDNAME: cannot determine hard link paths, skipped" 1>&2
                        continue
                fi
                hlinks+=("$OLDNAME")
@@ -455,13 +456,13 @@ lfs_migrate() {
                if ! $OPT_RSYNC; then
                        if $LFS migrate "${OPT_PASSTHROUGH[@]}" $layout \
                           "$OLDNAME"; then
-                               $ECHO "done migrate"
+                               $ECHO "done"
                                for link in ${hlinks[*]}; do
                                        add_to_set "$fid" "$link"
                                done
                                continue
                        elif $OPT_NO_RSYNC; then
-                               echo -e "$OLDNAME: refusing to fall back to rsync, skipped" 1>&2
+                               echo -e "\r$OLDNAME: refusing to fall back to rsync, skipped" 1>&2
                                continue
                        else
                                $ECHO -n "falling back to rsync: "
@@ -471,35 +472,35 @@ lfs_migrate() {
 
                NEWNAME=$(mktemp $UNLINK "$OLDNAME-lfs_migrate.tmp.XXXXXX")
                if [ $? -ne 0 -o -z "$NEWNAME" ]; then
-                       echo -e "$OLDNAME: cannot make temp file, skipped" 1>&2
+                       echo -e "\r$OLDNAME: cannot make temp file, skipped" 1>&2
                        continue
                fi
 
                if [ "$UNLINK" ]; then
                        if ! $LFS setstripe "${OPT_PASSTHROUGH[@]}" $layout \
                             "$NEWNAME"; then
-                               echo -e "\r\e[K$NEWNAME: setstripe failed, exiting" 1>&2
+                               echo -e "\r$NEWNAME: setstripe failed, exiting" 1>&2
                                exit 2
                        fi
                fi
 
                # we use --inplace, since we created our own temp file already
                if ! $RSYNC -a --inplace $RSYNC_OPTS "$OLDNAME" "$NEWNAME";then
-                       echo -e "$OLDNAME: copy error, exiting" 1>&2
+                       echo -e "\r$OLDNAME: copy error, exiting" 1>&2
                        exit 4
                fi
 
                if $OPT_CHECK && ! cmp -s "$OLDNAME" "$NEWNAME"; then
-                       echo -e "$NEWNAME: compare failed, exiting" 1>&2
+                       echo -e "\r$NEWNAME: compare failed, exiting" 1>&2
                        exit 8
                fi
 
                if ! mv "$NEWNAME" "$OLDNAME"; then
-                       echo -e "$OLDNAME: rename error, exiting" 1>&2
+                       echo -e "\r$OLDNAME: rename error, exiting" 1>&2
                        exit 12
                fi
 
-               $ECHO "done migrate via rsync"
+               $ECHO "done rsync"
                for link in ${hlinks[*]}; do
                        if [ "$link" != "$OLDNAME" ]; then
                                ln -f "$OLDNAME" "$link"

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index ba2daa7..7cb2c85 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -6312,7 +6312,7 @@ check_migrate_links() {
        # make sure hard links were properly detected, and migration was
        # performed only once for the entire link set; nonlinked files should
        # also be migrated
-       local actual=$(grep -c 'done migrate' <<< "$migrate_out")
+       local actual=$(grep -c 'done' <<< "$migrate_out")
        local expected=$(($uniq_count + 1))
 
        [ "$actual" -eq  "$expected" ] ||
@@ -6363,13 +6363,14 @@ test_56xc() {
 
        # Test 1: ensure file < 1 GB is always migrated with 1 stripe
        echo -n "Setting initial stripe for 20MB test file..."
-       $LFS setstripe -c 2 -i 0 "$dir/20mb" || error "cannot setstripe"
+       $LFS setstripe -c 2 -i 0 "$dir/20mb" ||
+               error "cannot setstripe 20MB file"
        echo "done"
        echo -n "Sizing 20MB test file..."
        truncate "$dir/20mb" 20971520 || error "cannot create 20MB test file"
        echo "done"
        echo -n "Verifying small file autostripe count is 1..."
-       $LFS_MIGRATE -y -A -C 1 "$dir/20mb" &> /dev/null ||
+       $LFS_MIGRATE -y -A -C 1 "$dir/20mb" ||
                error "cannot migrate 20MB file"
        local stripe_count=$($LFS getstripe -c "$dir/20mb") ||
                error "cannot get stripe for $dir/20mb"
@@ -6382,24 +6383,28 @@ test_56xc() {
        # sqrt(size_in_gb) + 1 OSTs but is larger than 1GB.  The file must
        # have at least an additional 1KB for each desired stripe for test 3
        echo -n "Setting stripe for 1GB test file..."
-       $LFS setstripe -c 1 -i 0 "$dir/1gb" || error "cannot setstripe"
+       $LFS setstripe -c 1 -i 0 "$dir/1gb" || error "cannot setstripe 1GB file"
        echo "done"
        echo -n "Sizing 1GB test file..."
        # File size is 1GB + 3KB
-       truncate "$dir/1gb" 1073744896 &> /dev/null ||
-               error "cannot create 1GB test file"
-       echo "done"
-       echo -n "Migrating 1GB file..."
-       $LFS_MIGRATE -y -A -C 1 "$dir/1gb" &> /dev/null ||
-               error "cannot migrate file"
-       echo "done"
-       echo -n "Verifying autostripe count is sqrt(n) + 1..."
-       stripe_count=$($LFS getstripe -c "$dir/1gb") ||
-               error "cannot get stripe for $dir/1gb"
-       [ $stripe_count -eq 2 ] ||
-               error "unexpected stripe count $stripe_count (expected 2)"
+       truncate "$dir/1gb" 1073744896 || error "cannot create 1GB test file"
        echo "done"
 
+       # need at least 512MB per OST for 1GB file to fit in 2 stripes
+       local avail=$($LCTL get_param -n llite.$FSNAME*.kbytesavail)
+       if (( avail > 524288 * OSTCOUNT )); then
+               echo -n "Migrating 1GB file..."
+               $LFS_MIGRATE -y -A -C 1 "$dir/1gb" ||
+                       error "cannot migrate 1GB file"
+               echo "done"
+               echo -n "Verifying autostripe count is sqrt(n) + 1..."
+               stripe_count=$($LFS getstripe -c "$dir/1gb") ||
+                       error "cannot getstripe for 1GB file"
+               [ $stripe_count -eq 2 ] ||
+                       error "unexpected stripe count $stripe_count != 2"
+               echo "done"
+       fi
+
        # Test 3: File is too large to fit within the available space on
        # sqrt(n) + 1 OSTs.  Simulate limited available space with -X
        if [ $OSTCOUNT -ge 3 ]; then
@@ -6407,15 +6412,15 @@ test_56xc() {
                # file size (1GB + 3KB) / OST count (3).
                local kb_per_ost=349526
 
-               echo -n "Migrating 1GB file..."
-               $LFS_MIGRATE -y -A -C 1 -X $kb_per_ost "$dir/1gb" &>> \
-                       /dev/null || error "cannot migrate file"
+               echo -n "Migrating 1GB file with limit..."
+               $LFS_MIGRATE -y -A -C 1 -X $kb_per_ost "$dir/1gb" ||
+                       error "cannot migrate 1GB file with limit"
                echo "done"
 
                stripe_count=$($LFS getstripe -c "$dir/1gb")
-               echo -n "Verifying autostripe count with limited space..."
-               [ "$stripe_count" -a $stripe_count -eq 3 ] ||
-                       error "unexpected stripe count $stripe_count (wanted 3)"
+               echo -n "Verifying 1GB autostripe count with limited space..."
+               [ "$stripe_count" -a $stripe_count -ge 3 ] ||
+                       error "unexpected stripe count $stripe_count (min 3)"
                echo "done"
        fi
 
@@ -6466,18 +6471,20 @@ test_56z() { # LU-4824
        test_mkdir $dir
        for i in d{0..9}; do
                test_mkdir $dir/$i
+               touch $dir/$i/$tfile
        done
-       touch $dir/d{0..9}/$tfile
        $LFS find $DIR/non_existent_dir $dir &&
                error "$LFS find did not return an error"
        # Make a directory unsearchable. This should NOT be the last entry in
        # directory order.  Arbitrarily pick the 6th entry
        chmod 700 $($LFS find $dir -type d | sed '6!d')
 
+       $RUNAS $LFS find $DIR/non_existent $dir
        local count=$($RUNAS $LFS find $DIR/non_existent $dir | wc -l)
 
        # The user should be able to see 10 directories and 9 files
-       [ $count == 19 ] || error "$LFS find did not continue after error"
+       (( count == 19 )) ||
+               error "$LFS find found $count != 19 entries after error"
 }
 run_test 56z "lfs find should continue after an error"