From 4ca74d8b31f7342b4a228109f3665c4584580f4b Mon Sep 17 00:00:00 2001 From: Liu Ying Date: Mon, 14 Jan 2013 23:10:45 +0800 Subject: [PATCH] LU-797 tests: fix ost-pools test timeout issues on b1_8 The test time of the ost-pools subtests is unreasonably long. test_14 fills an OST to 90% full, regardless of the OST size. Skip the test if the amount of data to be written is too large to run in a practical time. test_18 creates 3x3x30000 files to compare performance with/without pools enabled. Instead of creating a fixed number of files, use createmany to run for a specific (short) time to measure performance. test_23 tried to fill all OSTs 100% full. Split this test into two: - test_23a to test quota with a file in a pool - test_23b to test OOS with a file striped over pool The following patches are merged into this one: - LU-797 tests: speed up ost-pools tests (master patch eea698c944283b755882d8f504d2fcc8ea371bd8) - LU-797 tests: skip ost-pools.sh 23b when SLOW=no (master patch f7b4054cfc1d30fbbfd56acfe4b5a7a334de8212) - LU-797 tests: process lfs df output properly (master patch b1a1ec6300a5ec3925b725d5d2b783314dff3f8) - LU-797 tests: improve test_23b of ost-pools.sh (master patch 6dd41a43e3cdff1b2e0713cfc163734889d8650a) Test-Parameters: envdefinitions=SLOW=yes testlist=ost-pools Signed-off-by: Andreas Dilger Signed-off-by: Niu Yawei Signed-off-by: Liu Ying Change-Id: I391e641664890e7172d2ed1da815894e656826ce Reviewed-on: http://review.whamcloud.com/4898 Tested-by: Hudson Reviewed-by: Niu Yawei Reviewed-by: Jian Yu Tested-by: Maloo Reviewed-by: Johann Lombardi --- lustre/tests/ost-pools.sh | 889 ++++++++++++++++++++++----------------- lustre/tests/sanity-benchmark.sh | 2 +- lustre/tests/sanity-quota.sh | 8 +- lustre/tests/sanityn.sh | 2 +- lustre/tests/test-framework.sh | 20 +- 5 files changed, 511 insertions(+), 410 deletions(-) diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh index 8d7e09c..6380d9a 100644 --- a/lustre/tests/ost-pools.sh +++ b/lustre/tests/ost-pools.sh @@ -7,7 +7,7 @@ # # Run test by setting NOSETUP=true when ltest has setup env for us -SRCDIR=`dirname $0` +SRCDIR=$(dirname $0) export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/../utils:$PATH:/sbin ONLY=${ONLY:-"$*"} @@ -15,8 +15,8 @@ ALWAYS_EXCEPT="$OST_POOLS_EXCEPT" # bug number for skipped test: - # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! -[ "$ALWAYS_EXCEPT$EXCEPT" ] && \ - echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT`" +[ "$ALWAYS_EXCEPT$EXCEPT" ] && + echo "Skipping tests: $(echo $ALWAYS_EXCEPT $EXCEPT)" TMP=${TMP:-/tmp} ORIG_PWD=${PWD} @@ -29,28 +29,24 @@ init_logging check_and_setup_lustre +[ "$SLOW" = "no" ] && EXCEPT_SLOW="23b" + DIR=${DIR:-$MOUNT} assert_DIR build_test_filter -FAIL_ON_ERROR=${FAIL_ON_ERROR:-true} - -LFS=${LFS:-lfs} -LCTL=${LCTL:-lctl} SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"} GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"} SINGLEMDS=mds -TSTUSR=${TSTUSR:-"quota_usr"} -TSTID=${TSTID:-60000} -RUNAS="runas -u $TSTID -g $TSTID" +MAXFREE=${MAXFREE:-$((2000000 * OSTCOUNT))} # OST pools tests -POOL=${POOL:-pool1} -POOL2=${POOL2:-pool2} -POOL3=${POOL3:-pool3} +POOL=testpool +POOL2=${POOL2:-${POOL}2} +POOL3=${POOL3:-${POOL}3} NON_EXISTANT_POOL=nonexistantpool NON_EXISTANT_FS=nonexistantfs INVALID_POOL=some_invalid_pool_name @@ -79,7 +75,7 @@ create_dir() { else $SETSTRIPE -c $count -p $pool $dir fi - [[ $? -eq 0 ]] || \ + [[ $? -eq 0 ]] || error "$SETSTRIPE -p $pool $dir failed." } @@ -90,16 +86,16 @@ create_file() { local index=${4:-"-1"} rm -f $file $SETSTRIPE -o $index -c $count -p $pool $file - [[ $? -eq 0 ]] || \ + [[ $? -eq 0 ]] || error "$SETSTRIPE -p $pool $file failed." } osts_in_pool() { local pool=$1 local res - for i in $(do_facet $SINGLEMDS lctl pool_list $FSNAME.$pool | \ - grep -v "^Pool:" | sed -e 's/_UUID$//;s/^.*-OST//'); do - res="$res $(printf "%d" 0x$i)" + for i in $(do_facet $SINGLEMDS lctl pool_list $FSNAME.$pool | + grep -v "^Pool:" | sed -e 's/_UUID$//;s/^.*-OST//'); do + res="$res $(printf "%d" 0x$i)" done echo $res } @@ -107,14 +103,14 @@ osts_in_pool() { check_dir_in_pool() { local dir=$1 local pool=$2 - local res=$($GETSTRIPE $dir | grep "^stripe_count:" \ - | cut -d ':' -f 5 | tr -d "[:blank:]") - if [[ "$res" == "$pool" ]]; then - return 0 - else + local res=$($GETSTRIPE $dir | grep "^stripe_count:" | + cut -d ':' -f 5 | tr -d "[:blank:]") + if [[ "$res" != "$pool" ]]; then error found $res instead of $pool return 1 fi + + return 0 } check_file_in_pool() { @@ -123,34 +119,38 @@ check_file_in_pool() { } check_file_in_osts() { - local file=$1 - local pool_list=${2:-$TGT_LIST} - local count=$3 - local res=$($GETSTRIPE $file | grep 0x | cut -f2) - local i - for i in $res - do - found=$(echo :$pool_list: | tr " " ":" | grep :$i:) - if [[ "$found" == "" ]]; then - echo "pool list: $pool_list" - echo "striping: $res" - $GETSTRIPE $file - error "$file not allocated from OSTs $pool_list." - return 1 - fi - done + local file=$1 + local pool_list=${2:-$TGT_LIST} + local count=$3 + local res=$($GETSTRIPE $file | grep 0x | cut -f2) + local i + for i in $res; do + found=$(echo :$pool_list: | tr " " ":" | grep :$i:) + if [[ "$found" == "" ]]; then + echo "pool list: $pool_list" + echo "striping: $res" + $GETSTRIPE $file + error "$file not allocated from OSTs $pool_list." + return 1 + fi + done - local ost_count=$($GETSTRIPE $file | grep 0x | wc -l) - [[ -n "$count" ]] && [[ $ost_count -ne $count ]] && \ - { error "Stripe count $count expected; got $ost_count" && return 1; } + local ost_count=$($GETSTRIPE $file | grep 0x | wc -l) + [[ -n "$count" ]] && [[ $ost_count -ne $count ]] && + { error "Stripe count $count expected; got $ost_count" && return 1;} - return 0 + return 0 +} + +file_pool() { + $GETSTRIPE -v $1 | grep "^lmm_pool:" | tr -d "[:blank:]" | cut -f 2 -d ':' } check_file_not_in_pool() { local file=$1 local pool=$2 - local res=$($GETSTRIPE -v $file | grep "^pool:" | tr -d "[:blank:]" | cut -f 2 -d ':') + local res=$(file_pool $file) + if [[ "$res" == "$pool" ]]; then error "File $file is in pool: $res" return 1 @@ -162,58 +162,54 @@ check_file_not_in_pool() { check_dir_not_in_pool() { local dir=$1 local pool=$2 - local res=$($GETSTRIPE -v $dir | grep "^stripe_count" | head -1 | \ - cut -f 8 -d ' ') + local res=$($GETSTRIPE -v $dir | grep "^stripe_count" | head -1 | + cut -f 8 -d ' ') if [[ "$res" == "$pool" ]]; then error "File $dir is in pool: $res" return 1 - else - return 0 fi + return 0 } drain_pool() { pool=$1 - wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool" ""\ - ||error "Failed to remove targets from pool: $pool" + wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool" "" || + error "Failed to remove targets from pool: $pool" } add_pool() { local pool=$1 local osts=$2 - local tgt="${3}$(lctl get_param -n lov.$FSNAME-*.pools.$pool | \ - sort -u | tr '\n' ' ')" + local tgt="${3}$(lctl get_param -n lov.$FSNAME-*.pools.$pool | + sort -u | tr '\n' ' ')" do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool $osts local RC=$? [[ $RC -ne 0 ]] && return $RC - wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool | \ - sort -u | tr '\n' ' ' " "$tgt" || RC=1 + wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$pool | + sort -u | tr '\n' ' ' " "$tgt" >/dev/null || RC=1 [[ $RC -ne 0 ]] && error "pool_add failed: $1; $2" return $RC } create_pool_nofail() { create_pool $FSNAME.$1 - if [[ $? != 0 ]] - then + if [[ $? != 0 ]]; then error "Pool creation of $1 failed" fi } create_pool_fail() { create_pool $FSNAME.$1 - if [[ $? == 0 ]] - then + if [[ $? == 0 ]]; then error "Pool creation of $1 succeeded; should have failed" fi } cleanup_tests() { # Destroy pools from previous test runs - for p in $(do_facet $SINGLEMDS lctl pool_list $FSNAME | \ - grep $FSNAME.pool[0-$OSTCOUNT]); do + for p in $(do_facet $SINGLEMDS lctl pool_list $FSNAME | grep $POOL); do destroy_pool_int $p; done rm -rf $DIR/d0.${TESTSUITE} @@ -223,110 +219,157 @@ ost_pools_init() { cleanup_tests } +set_cleanup_trap() { + trap "cleanup_tests $FSNAME" EXIT +} # Initialization remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0 ost_pools_init -trap "cleanup_pools $FSNAME" EXIT # Tests for new commands added -test_1() { - echo "Creating a pool with a 1 character pool name" +test_1a() { + set_cleanup_trap create_pool_nofail p + destroy_pool p +} +run_test 1a "Create a pool with a 1 character pool name" - echo "Creating a pool with a 10 character pool name" - create_pool_nofail p123456789 - destroy_pool p123456789 +test_1b() { + set_cleanup_trap + create_pool_nofail ${POOL}12 + destroy_pool ${POOL}12 +} +run_test 1b "Create a pool with a 10 character pool name" - echo "Creating a pool with a 16 character pool name" - create_pool_nofail p123456789123456 - destroy_pool p123456789123456 +test_1c() { + set_cleanup_trap + create_pool_nofail ${POOL}12345678 + destroy_pool ${POOL}12345678 +} +run_test 1c "Create a pool with a 16 character pool name" - echo "Creating a pool with a 17 character pool name; should fail" - create_pool_fail p1234567891234567 +test_1d() { + set_cleanup_trap + create_pool_fail ${POOL}123456789 +} +run_test 1d "Create a pool with a 17 char pool name; should fail" - echo "Creating a pool with a 1000 character pool name; should fail" - local pool_name="p" - for ((i=1;i<=999;i++)); do pool_name=${pool_name}"o"; done +test_1e() { + set_cleanup_trap + local pool_name="$POOL" + for ((i = 1; i <= 991; i++)); do pool_name=${pool_name}"o"; done create_pool_fail $pool_name +} +run_test 1e "Create a pool with a 1000 char pool name; should fail" - echo "pool_new should fail if fs-name or poolname are missing." - do_facet $SINGLEMDS lctl pool_new .pool1 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_new did not fail even though fs-name was missing." - do_facet $SINGLEMDS lctl pool_new pool1 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_new did not fail even though fs-name was missing." +test_1f() { + set_cleanup_trap + do_facet $SINGLEMDS lctl pool_new .$POOL 2>/dev/null + [[ $? -ne 0 ]] || + error "pool_new did not fail even though fs-name was missing" +} +run_test 1f "pool_new should fail if fs-name is missing" + +test_1g() { + set_cleanup_trap + do_facet $SINGLEMDS lctl pool_new $POOL 2>/dev/null + [[ $? -ne 0 ]] || + error "pool_new did not fail even though fs-name was missing" +} +run_test 1g "pool_new should fail if fs-name is missing" + +test_1h() { + set_cleanup_trap do_facet $SINGLEMDS lctl pool_new ${FSNAME}. 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_new did not fail even though pool name was missing." + [[ $? -ne 0 ]] || + error "pool_new did not fail even though pool name was missing" +} +run_test 1h "pool_new should fail if poolname is missing" + +test_1i() { + set_cleanup_trap do_facet $SINGLEMDS lctl pool_new . 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_new did not fail even though pool name and fs-name " \ - "were missing." - do_facet $SINGLEMDS lctl pool_new ${FSNAME},pool1 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_new did not fail even though pool name format was wrong" - do_facet $SINGLEMDS lctl pool_new ${FSNAME}/pool1 2>/dev/null - [[ $? -ne 0 ]] || \ + [[ $? -ne 0 ]] || + error "pool_new did not fail even if pool and fs-name were missing" +} +run_test 1i "pool_new should fail if poolname and fs-name are missing" + +test_1j() { + set_cleanup_trap + do_facet $SINGLEMDS lctl pool_new ${FSNAME},$POOL 2>/dev/null + [[ $? -ne 0 ]] || error "pool_new did not fail even though pool name format was wrong" +} +run_test 1j "pool_new should fail if poolname format is wrong" - do_facet $SINGLEMDS lctl pool_new ${FSNAME}.p 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_new did not fail even though pool1 existed" - destroy_pool p +test_1k() { + set_cleanup_trap + do_facet $SINGLEMDS lctl pool_new ${FSNAME}/$POOL 2>/dev/null + [[ $? -ne 0 ]] || + error "pool_new did not fail even though pool name format was wrong" +} +run_test 1k "pool_new should fail if poolname format is wrong" +test_1m() { + set_cleanup_trap + create_pool_nofail $POOL2 + do_facet $SINGLEMDS lctl pool_new ${FSNAME}.$POOL2 2>/dev/null + [[ $? -ne 0 ]] || + error "pool_new did not fail even though $POOL2 existed" + destroy_pool $POOL2 } -run_test 1 "Test lctl pool_new =========================================" +run_test 1m "pool_new did not fail even though $POOL2 existed" test_2a() { + set_cleanup_trap destroy_pool $POOL do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null - [[ $? -ne 0 ]] || \ - error " pool_add did not fail even though pool did " \ - " not exist." + [[ $? -ne 0 ]] || + error "pool_add did not fail even though $POOL did not exist" } -run_test 2a "pool_add: non-existant pool" +run_test 2a "pool_add: non-existant pool $POOL" test_2b() { - do_facet $SINGLEMDS lctl pool_add $FSNAME.p1234567891234567890 \ + set_cleanup_trap + do_facet $SINGLEMDS lctl pool_add $FSNAME.${POOL}1234567890 \ $FSNAME-OST0000 2>/dev/null - [[ $? -ne 0 ]] || \ + [[ $? -ne 0 ]] || error "pool_add did not fail even though pool name was invalid." } run_test 2b "pool_add: Invalid pool name" # Testing various combinations of OST name list test_2c() { + set_cleanup_trap local TGT local RC lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - destroy_pool $POOL + [[ $? -ne 0 ]] || destroy_pool $POOL create_pool_nofail $POOL # 1. OST0000 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL OST0000 - RC=$?; [[ $RC -eq 0 ]] || \ + RC=$?; [[ $RC -eq 0 ]] || error "pool_add failed. $FSNAME $POOL OST0000: $RC" do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL OST0000 drain_pool $POOL # 2. lustre-OST0000 do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000 - RC=$?; [[ $RC -eq 0 ]] || \ + RC=$?; [[ $RC -eq 0 ]] || error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000: $RC" do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000 drain_pool $POOL # 3. lustre-OST0000_UUID do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000_UUID - RC=$?; [[ $RC -eq 0 ]] || \ + RC=$?; [[ $RC -eq 0 ]] || error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000_UUID: $RC" do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000_UUID drain_pool $POOL @@ -336,169 +379,169 @@ test_2c() { for i in $TGT_LIST; do TGT=${TGT}$(printf "%04x," $i); done TGT="${TGT}]" do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT - [[ $? -eq 0 ]] || \ - error "pool_add failed. $FSNAME.$POOL $TGT. $RC" + [[ $? -eq 0 ]] || error "pool_add failed. $FSNAME.$POOL $TGT. $RC" do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT drain_pool $POOL # 5. lustre-OST[0-5/1] do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT_ALL - RC=$?; [[ $RC -eq 0 ]] || \ + RC=$?; [[ $RC -eq 0 ]] || error "pool_add failed. $FSNAME $POOL" "$TGT_ALL $RC" - wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | \ + wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | sort -u | tr '\n' ' ' " "$TGT_UUID" || error "Add to pool failed" do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT_ALL drain_pool $POOL destroy_pool $POOL } -run_test 2c "pool_add: OST index combinations ===========================" +run_test 2c "pool_add: OST index combinations" test_2d() { + set_cleanup_trap local TGT local RC lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - destroy_pool $POOL + [[ $? -ne 0 ]] || destroy_pool $POOL create_pool_nofail $POOL TGT=$(printf "$FSNAME-OST%04x_UUID " $OSTCOUNT) do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT - RC=$?; [[ $RC -ne 0 ]] || \ + RC=$?; [[ $RC -ne 0 ]] || error "pool_add succeeded for an OST ($TGT) that does not exist." destroy_pool $POOL } -run_test 2d "pool_add: OSTs that don't exist should be rejected ========" +run_test 2d "pool_add: OSTs that don't exist should be rejected" test_2e() { + set_cleanup_trap local TGT local RC local RESULT $LCTL get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - destroy_pool $POOL + [[ $? -ne 0 ]] || destroy_pool $POOL create_pool_nofail $POOL TGT="$FSNAME-OST0000_UUID " do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT - wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | \ - sort -u | tr '\n' ' ' " "$TGT" || error "Add to pool failed" + wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL | + sort -u | tr '\n' ' ' " "$TGT" || error "Add to pool failed" RESULT=$(do_facet $SINGLEMDS \ - "LOCALE=C $LCTL pool_add $FSNAME.$POOL $TGT 2>&1") + "LOCALE=C $LCTL pool_add $FSNAME.$POOL $TGT 2>&1") RC=$? echo $RESULT - [[ $RC -ne 0 ]] || \ + [[ $RC -ne 0 ]] || error "pool_add succeeded for an OST that was already in the pool." - [[ $(grep "already in pool" <<< $RESULT) ]] || \ + [[ $(grep "already in pool" <<< $RESULT) ]] || error "pool_add failed as expected but error message not as expected." destroy_pool $POOL } -run_test 2e "pool_add: OST already in a pool should be rejected ========" +run_test 2e "pool_add: OST already in a pool should be rejected" test_3a() { + set_cleanup_trap lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - destroy_pool $POOL + [[ $? -ne 0 ]] || destroy_pool $POOL - do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null - [[ $? -ne 0 ]] || \ + do_facet $SINGLEMDS \ + lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null + [[ $? -ne 0 ]] || error "pool_remove did not fail even though pool did not exist." } run_test 3a "pool_remove: non-existant pool" test_3b() { - do_facet $SINGLEMDS lctl pool_remove ${NON_EXISTANT_FS}.$POOL OST0000 2>/dev/null - [[ $? -ne 0 ]] || \ + set_cleanup_trap + do_facet $SINGLEMDS \ + lctl pool_remove ${NON_EXISTANT_FS}.$POOL OST0000 2>/dev/null + [[ $? -ne 0 ]] || error "pool_remove did not fail even though fsname did not exist." } run_test 3b "pool_remove: non-existant fsname" test_3c() { + set_cleanup_trap do_facet $SINGLEMDS lctl pool_remove $FSNAME.p1234567891234567890 \ $FSNAME-OST0000 2>/dev/null - [[ $? -ne 0 ]] || \ + [[ $? -ne 0 ]] || error "pool_remove did not fail even though pool name was invalid." } run_test 3c "pool_remove: Invalid pool name" # Testing various combinations of OST name list test_3d() { + set_cleanup_trap lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - destroy_pool $POOL + [[ $? -ne 0 ]] || destroy_pool $POOL create_pool_nofail $POOL do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL OST0000 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL OST0000 - [[ $? -eq 0 ]] || \ - error "pool_remove failed. $FSNAME $POOL OST0000" + [[ $? -eq 0 ]] || error "pool_remove failed. $FSNAME $POOL OST0000" drain_pool $POOL do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000 do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000 - [[ $? -eq 0 ]] || \ - error "pool_remove failed. $FSNAME $POOL $FSNAME-OST0000" + [[ $? -eq 0 ]] || error "pool_remove failed. $FSNAME $POOL $FSNAME-OST0000" drain_pool $POOL do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000_UUID do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000_UUID - [[ $? -eq 0 ]] || \ + [[ $? -eq 0 ]] || error "pool_remove failed. $FSNAME $POOL $FSNAME-OST0000_UUID" drain_pool $POOL add_pool $POOL $TGT_ALL "$TGT_UUID" do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT_ALL - [[ $? -eq 0 ]] || \ - error "pool_remove failed. $FSNAME $POOL" $TGT_ALL + [[ $? -eq 0 ]] || error "pool_remove failed. $FSNAME $POOL" $TGT_ALL drain_pool $POOL destroy_pool $POOL } -run_test 3d "pool_remove: OST index combinations ===========================" +run_test 3d "pool_remove: OST index combinations" test_4a() { + set_cleanup_trap lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - destroy_pool $POOL + [[ $? -ne 0 ]] || destroy_pool $POOL do_facet $SINGLEMDS lctl pool_destroy $FSNAME.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ + [[ $? -ne 0 ]] || error "pool_destroy did not fail even though pool did not exist." } run_test 4a "pool_destroy: non-existant pool" test_4b() { + set_cleanup_trap do_facet $SINGLEMDS lctl pool_destroy ${NON_EXISTANT_FS}.$POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_destroy did not fail even though the filesystem did not exist." + [[ $? -ne 0 ]] || + error "pool_destroy did not fail even though filesystem did not exist." } run_test 4b "pool_destroy: non-existant fs-name" test_4c() { + set_cleanup_trap create_pool_nofail $POOL add_pool $POOL "OST0000" "$FSNAME-OST0000_UUID " do_facet $SINGLEMDS lctl pool_destroy ${FSNAME}.$POOL - [[ $? -ne 0 ]] || \ - error "pool_destroy succeeded with a non-empty pool." + [[ $? -ne 0 ]] || error "pool_destroy succeeded with a non-empty pool." destroy_pool $POOL } -run_test 4c "pool_destroy: non-empty pool ===============================" +run_test 4c "pool_destroy: non-empty pool" sub_test_5() { local LCMD=$1 $LCMD pool_list 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_list did not fail even though fsname was not mentioned." + [[ $? -ne 0 ]] || error "pool_list did not fail even though fsname missing." destroy_pool $POOL 2>/dev/null destroy_pool $POOL2 2>/dev/null @@ -506,38 +549,32 @@ sub_test_5() { create_pool_nofail $POOL create_pool_nofail $POOL2 $LCMD pool_list $FSNAME - [[ $? -eq 0 ]] || \ - error "pool_list $FSNAME failed." + [[ $? -eq 0 ]] || error "pool_list $FSNAME failed." do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT_ALL $LCMD pool_list $FSNAME.$POOL - [[ $? -eq 0 ]] || \ - error "pool_list $FSNAME.$POOL failed." + [[ $? -eq 0 ]] || error "pool_list $FSNAME.$POOL failed." $LCMD pool_list ${NON_EXISTANT_FS} 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_list did not fail for a non-existant fsname $NON_EXISTANT_FS" + [[ $? -ne 0 ]] || error "pool_list did not fail for fsname $NON_EXISTANT_FS" $LCMD pool_list ${FSNAME}.$NON_EXISTANT_POOL 2>/dev/null - [[ $? -ne 0 ]] || \ - error "pool_list did not fail for a non-existant pool $NON_EXISTANT_POOL" + [[ $? -ne 0 ]] || error "pool_list did not fail for pool $NON_EXISTANT_POOL" if [[ ! $(grep $SINGLEMDS <<< $LCMD) ]]; then echo $LCMD pool_list $DIR $LCMD pool_list $DIR - [[ $? -eq 0 ]] || \ - error "pool_list failed for $DIR" + [[ $? -eq 0 ]] || error "pool_list failed for $DIR" mkdir -p ${DIR}/d1 $LCMD pool_list ${DIR}/d1 - [[ $? -eq 0 ]] || \ - error "pool_list failed for ${DIR}/d1" + [[ $? -eq 0 ]] || error "pool_list failed for ${DIR}/d1" fi rm -rf ${DIR}nonexistant $LCMD pool_list ${DIR}nonexistant 2>/dev/null - [[ $? -ne 0 ]] || \ + [[ $? -ne 0 ]] || error "pool_list did not fail for invalid mountpoint ${DIR}nonexistant" destroy_pool $POOL @@ -545,6 +582,7 @@ sub_test_5() { } test_5() { + set_cleanup_trap # Issue commands from client sub_test_5 $LCTL sub_test_5 $LFS @@ -557,6 +595,7 @@ test_5() { run_test 5 "lfs/lctl pool_list" test_6() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local POOL_DIR=$POOL_ROOT/dir_tst local POOL_FILE=$POOL_ROOT/file_tst @@ -564,41 +603,35 @@ test_6() { create_pool_nofail $POOL do_facet $SINGLEMDS lctl pool_list $FSNAME - [[ $? -eq 0 ]] || \ - error "pool_list $FSNAME failed." + [[ $? -eq 0 ]] || error "pool_list $FSNAME failed." add_pool $POOL $TGT_ALL "$TGT_UUID" mkdir -p $POOL_DIR $SETSTRIPE -c -1 -p $POOL $POOL_DIR - [[ $? -eq 0 ]] || \ - error "$SETSTRIPE -p $POOL failed." + [[ $? -eq 0 ]] || error "$SETSTRIPE -p $POOL failed." check_dir_in_pool $POOL_DIR $POOL # If an invalid pool name is specified, the command should fail $SETSTRIPE -c 2 -p $INVALID_POOL $POOL_DIR 2>/dev/null - [[ $? -ne 0 ]] || \ - error "setstripe to invalid pool did not fail." + [[ $? -ne 0 ]] || error "setstripe to invalid pool did not fail." # If the pool name does not exist, the command should fail $SETSTRIPE -c 2 -p $NON_EXISTANT_POOL $POOL_DIR 2>/dev/null - [[ $? -ne 0 ]] || \ - error "setstripe to non-existant pool did not fail." + [[ $? -ne 0 ]] || error "setstripe to non-existant pool did not fail." # lfs setstripe should work as before if a pool name is not specified. $SETSTRIPE -c -1 $POOL_DIR - [[ $? -eq 0 ]] || \ - error "$SETSTRIPE -p $POOL_DIR failed." + [[ $? -eq 0 ]] || error "$SETSTRIPE -p $POOL_DIR failed." $SETSTRIPE -c -1 $POOL_FILE - [[ $? -eq 0 ]] || \ - error "$SETSTRIPE -p $POOL_FILE failed." + [[ $? -eq 0 ]] || error "$SETSTRIPE -p $POOL_FILE failed." # lfs setstripe should fail if a start index that is outside the # pool is specified. create_pool_nofail $POOL2 add_pool $POOL2 "OST0000" "$FSNAME-OST0000_UUID " $SETSTRIPE -o 1 -p $POOL2 $ROOT_POOL/$tfile 2>/dev/null - [[ $? -ne 0 ]] || \ + [[ $? -ne 0 ]] || error "$SETSTRIPE with start index outside the pool did not fail." destroy_pool $POOL @@ -607,14 +640,15 @@ test_6() { run_test 6 "getstripe/setstripe" test_11() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} - [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return + [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return create_pool_nofail $POOL create_pool_nofail $POOL2 - local start=$(printf %04x $((TGT_FIRST+1))) + local start=$(printf %04x $((TGT_FIRST + 1))) do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \ $FSNAME-OST[$start-$TGT_MAX/2] @@ -626,14 +660,14 @@ test_11() { check_dir_in_pool $POOL_ROOT/dir1 $POOL local numfiles=100 - createmany -o $POOL_ROOT/dir1/$tfile $numfiles || \ + createmany -o $POOL_ROOT/dir1/$tfile $numfiles || error "createmany $POOL_ROOT/dir1/$tfile failed!" for file in $POOL_ROOT/dir1/*; do check_file_in_pool $file $POOL done - createmany -o $POOL_ROOT/dir2/$tfile $numfiles || \ + createmany -o $POOL_ROOT/dir2/$tfile $numfiles || error "createmany $POOL_ROOT/dir2/$tfile failed!" for file in $POOL_ROOT/dir2/*; do check_file_in_pool $file $POOL2 @@ -649,14 +683,15 @@ test_11() { run_test 11 "OSTs in overlapping/multiple pools" test_12() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} - [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return + [[ $OSTCOUNT -le 2 ]] && skip_env "Need at least 3 OSTs" && return create_pool_nofail $POOL create_pool_nofail $POOL2 - local start=$(printf %04x $((TGT_FIRST+1))) + local start=$(printf %04x $((TGT_FIRST + 1))) do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \ $FSNAME-OST[$start-$TGT_MAX/2] @@ -706,7 +741,8 @@ test_12() { run_test 12 "OST Pool Membership" test_13() { - [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return + set_cleanup_trap + [[ $OSTCOUNT -le 2 ]] && skip_env "Need at least 3 OSTs" && return local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local numfiles=10 @@ -716,52 +752,52 @@ test_13() { add_pool $POOL $TGT_ALL "$TGT_UUID" create_dir $POOL_ROOT/dir1 $POOL -1 - createmany -o $POOL_ROOT/dir1/$tfile $numfiles || \ + createmany -o $POOL_ROOT/dir1/$tfile $numfiles || error "createmany $POOL_ROOT/dir1/$tfile failed!" for file in $POOL_ROOT/dir1/*; do check_file_in_pool $file $POOL $OSTCOUNT done create_file $POOL_ROOT/dir1/file1 $POOL 1 $TGT_FIRST - create_file $POOL_ROOT/dir1/file2 $POOL 1 $((TGT_FIRST+1)) - create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST+2)) + create_file $POOL_ROOT/dir1/file2 $POOL 1 $((TGT_FIRST + 1)) + create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST + 2)) check_file_in_pool $POOL_ROOT/dir1/file1 $POOL 1 check_file_in_pool $POOL_ROOT/dir1/file2 $POOL 1 - create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST+2)) + create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST + 2)) check_file_in_osts $POOL_ROOT/dir1/file1 $((16#$TGT_FIRST)) - check_file_in_osts $POOL_ROOT/dir1/file2 "$((TGT_FIRST+1))" - check_file_in_osts $POOL_ROOT/dir1/file3 "$((TGT_FIRST+2))" + check_file_in_osts $POOL_ROOT/dir1/file2 "$((TGT_FIRST + 1))" + check_file_in_osts $POOL_ROOT/dir1/file3 "$((TGT_FIRST + 2))" create_dir $POOL_ROOT/dir2 $POOL $count - createmany -o $POOL_ROOT/dir2/$tfile $numfiles || \ - error "createmany $POOL_ROOT/dir2/$tfile failed!" + createmany -o $POOL_ROOT/dir2/$tfile- $numfiles || + error "createmany $POOL_ROOT/dir2/$tfile- failed!" for file in $POOL_ROOT/dir2/*; do check_file_in_pool $file $POOL $count done - create_dir $POOL_ROOT/dir3 $POOL $count $((TGT_FIRST+1)) - createmany -o $POOL_ROOT/dir3/$tfile_ $numfiles || \ - error "createmany $POOL_ROOT/dir3/$tfile_ failed!" + create_dir $POOL_ROOT/dir3 $POOL $count $((TGT_FIRST + 1)) + createmany -o $POOL_ROOT/dir3/$tfile- $numfiles || + error "createmany $POOL_ROOT/dir3/$tfile- failed!" for file in $POOL_ROOT/dir3/*; do check_file_in_pool $file $POOL $count done create_dir $POOL_ROOT/dir4 $POOL 1 - createmany -o $POOL_ROOT/dir4/$tfile_ $numfiles || \ - error "createmany $POOL_ROOT/dir4/$tfile_ failed!" + createmany -o $POOL_ROOT/dir4/$tfile- $numfiles || + error "createmany $POOL_ROOT/dir4/$tfile- failed!" for file in $POOL_ROOT/dir4/*; do check_file_in_pool $file $POOL 1 done - create_dir $POOL_ROOT/dir5 $POOL 1 $((TGT_FIRST+2)) - createmany -o $POOL_ROOT/dir5/$tfile_ $numfiles || \ - error "createmany $POOL_ROOT/dir5/$tfile_ failed!" + create_dir $POOL_ROOT/dir5 $POOL 1 $((TGT_FIRST + 2)) + createmany -o $POOL_ROOT/dir5/$tfile- $numfiles || + error "createmany $POOL_ROOT/dir5/$tfile- failed!" for file in $POOL_ROOT/dir5/*; do check_file_in_pool $file $POOL 1 - check_file_in_osts $file "$((TGT_FIRST+2))" + check_file_in_osts $file "$((TGT_FIRST + 2))" done - rm -rf create_dir $POOL_ROOT/dir? + rm -rf $POOL_ROOT/dir[1-5]/ destroy_pool $POOL return 0 @@ -769,15 +805,16 @@ test_13() { run_test 13 "Striping characteristics in a pool" test_14() { - [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return + set_cleanup_trap + [[ $OSTCOUNT -le 2 ]] && skip_env "Need at least 3 OSTs" && return local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local numfiles=100 local i - # Create a new filesystem that is guaranteed to be balanced. - formatall - setupall + [ $OSTSIZE -gt $((MAXFREE / OSTCOUNT)) ] && + skip_env "OST size $OSTSIZE is larger than $((MAXFREE / OSTCOUNT))" && + return 0 create_pool_nofail $POOL create_pool_nofail $POOL2 @@ -787,39 +824,38 @@ test_14() { create_dir $POOL_ROOT/dir1 $POOL 1 create_file $POOL_ROOT/dir1/file $POOL 1 - local OST=$($GETSTRIPE $POOL_ROOT/dir1/file | grep 0x | cut -f2) + local OST=$($GETSTRIPE -i $POOL_ROOT/dir1/file) i=0 - while [[ $i -lt $numfiles ]]; - do - OST=$((OST+2)) + while [[ $i -lt $numfiles ]]; do + OST=$((OST + 2)) [[ $OST -gt $((16#$TGT_MAX)) ]] && OST=$TGT_FIRST # echo "Iteration: $i OST: $OST" create_file $POOL_ROOT/dir1/file${i} $POOL 1 check_file_in_pool $POOL_ROOT/dir1/file${i} $POOL - i=$((i+1)) + i=$((i + 1)) done # Fill up OST0 until it is nearly full. # Create 9 files of size OST0_SIZE/10 each. create_dir $POOL_ROOT/dir2 $POOL2 1 $LFS df $POOL_ROOT/dir2 - echo "Filling up OST0" - OST0_SIZE=`$LFS df $POOL_ROOT/dir2 | awk '/\[OST:0\]/ {print $4}'` + OST0_SIZE=$($LFS df $POOL_ROOT/dir2 | awk '/\[OST:0\]/ { print $4 }') FILE_SIZE=$((OST0_SIZE/1024/10)) + echo "Filling OST0 with 9 files of ${FILE_SIZE}MB in $POOL_ROOT/dir2" i=1 - while [[ $i -lt 10 ]]; - do - dd if=/dev/zero of=$POOL_ROOT/dir2/f${i} bs=1M count=$FILE_SIZE - i=$((i+1)) + while [[ $i -lt 10 ]]; do + dd if=/dev/zero of=$POOL_ROOT/dir2/f${i} bs=1M count=$FILE_SIZE + i=$((i + 1)) done + sleep 1 # get new statfs info $LFS df $POOL_ROOT/dir2 # OST $TGT_FIRST is no longer favored; but it may still be used. create_dir $POOL_ROOT/dir3 $POOL 1 create_file $POOL_ROOT/dir3/file $POOL 1 - createmany -o $POOL_ROOT/dir3/$tfile_ $numfiles || \ - error "createmany $POOL_ROOT/dir3/$tfile_ failed!" + createmany -o $POOL_ROOT/dir3/$tfile- $numfiles || + error "createmany $POOL_ROOT/dir3/$tfile- failed!" for file in $POOL_ROOT/dir3/*; do check_file_in_pool $file $POOL done @@ -827,37 +863,37 @@ test_14() { rm -rf $POOL_ROOT destroy_pool $POOL destroy_pool $POOL2 + return 0 } run_test 14 "Round robin and QOS striping within a pool" test_15() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local numfiles=100 local i=0 - while [[ $i -lt $OSTCOUNT ]] - do - create_pool_nofail pool${i} + while [[ $i -lt $OSTCOUNT ]]; do + create_pool_nofail $POOL${i} - local tgt=$(printf "$FSNAME-OST%04x_UUID " $i) - add_pool pool${i} "$FSNAME-OST[$(printf %04x $i)]" "$tgt" - create_dir $POOL_ROOT/dir${i} pool${i} - createmany -o $POOL_ROOT/dir$i/$tfile $numfiles || \ - error "createmany $POOL_ROOT/dir$i/$tfile failed!" + local tgt=$(printf "$FSNAME-OST%04x_UUID " $i) + add_pool $POOL${i} "$FSNAME-OST[$(printf %04x $i)]" "$tgt" + create_dir $POOL_ROOT/dir${i} $POOL${i} + createmany -o $POOL_ROOT/dir$i/$tfile $numfiles || + error "createmany $POOL_ROOT/dir$i/$tfile failed!" - for file in $POOL_ROOT/dir$i/*; do - check_file_in_osts $file $i - done + for file in $POOL_ROOT/dir$i/*; do + check_file_in_osts $file $i + done - i=$((i+1)) + i=$((i + 1)) done i=0 - while [[ $i -lt $OSTCOUNT ]] - do - destroy_pool pool${i} - i=$((i+1)) + while [[ $i -lt $OSTCOUNT ]]; do + destroy_pool $POOL${i} + i=$((i + 1)) done return 0 @@ -865,6 +901,7 @@ test_15() { run_test 15 "One directory per OST/pool" test_16() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local numfiles=10 local i=0 @@ -881,8 +918,8 @@ test_16() { done mkdir -p $dir - createmany -o $dir/$tfile $numfiles || \ - error "createmany $dir/$tfile failed!" + createmany -o $dir/$tfile $numfiles || + error "createmany $dir/$tfile failed!" for file in $dir/*; do check_file_in_pool $file $POOL @@ -897,6 +934,7 @@ test_16() { run_test 16 "Inheritance of pool properties" test_17() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local numfiles=10 local i=0 @@ -908,8 +946,8 @@ test_17() { local dir=$POOL_ROOT/dir create_dir $dir $POOL - createmany -o $dir/${tfile}1_ $numfiles || \ - error "createmany $dir/${tfile}1_ failed!" + createmany -o $dir/${tfile}1_ $numfiles || + error "createmany $dir/${tfile}1_ failed!" for file in $dir/*; do check_file_in_pool $file $POOL @@ -917,7 +955,7 @@ test_17() { destroy_pool $POOL - createmany -o $dir/${tfile}2_ $numfiles || \ + createmany -o $dir/${tfile}2_ $numfiles || error "createmany $dir/${tfile}2_ failed!" rm -rf $dir @@ -927,78 +965,79 @@ run_test 17 "Referencing an empty pool" create_perf() { local cdir=$1/d - local numfiles=$2 + local numsec=$2 local time mkdir -p $cdir - sync; sleep 5 # give pending IO a chance to go to disk - stat=$(createmany -o $cdir/${tfile} $numfiles | tail -1) - rm -rf $cdir sync - time=$(echo $stat | cut -f 5 -d ' ') - echo $stat >> /dev/stderr - echo $time + wait_delete_completed >/dev/null # give pending IO a chance to go to disk + stat=$(createmany -o $cdir/${tfile} -$numsec | tail -1) + files=$(echo $stat | cut -f 2 -d ' ') + echo $stat 1>&2 + unlinkmany $cdir/${tfile} $files > /dev/null + sync + + echo $files } test_18() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} - local numfiles=9877 + local numsec=15 + local iter=3 local plaindir=$POOL_ROOT/plaindir local pooldir=$POOL_ROOT/pooldir - local t1=0 - local t2=0 - local t3=0 + local f1=0 + local f2=0 + local f3=0 local diff - for i in $(seq 1 3); - do - echo "Create performance, iteration $i, $numfiles files x 3" + for i in $(seq 1 $iter); do + echo "Create performance, iteration $i, $numsec seconds x 3" - time1=$(create_perf $plaindir $numfiles) - echo "iter $i: $numfiles creates without pool: $time1" - t1=$(echo "scale=2; $t1 + $time1" | bc) + files1=$(create_perf $plaindir $numsec) + echo "iter $i: $files1 creates without pool" + f1=$(($f1 + $files1)) create_pool_nofail $POOL > /dev/null add_pool $POOL $TGT_ALL "$TGT_UUID" > /dev/null create_dir $pooldir $POOL - time2=$(create_perf $pooldir $numfiles) - echo "iter $i: $numfiles creates with pool: $time2" - t2=$(echo "scale=2; $t2 + $time2" | bc) + files2=$(create_perf $pooldir $numsec) + echo "iter $i: $files2 creates with pool" + f2=$(($f2 + $files2)) destroy_pool $POOL > /dev/null - time3=$(create_perf $pooldir $numfiles) - echo "iter $i: $numfiles creates with missing pool: $time3" - t3=$(echo "scale=2; $t3 + $time3" | bc) + files3=$(create_perf $pooldir $numsec) + echo "iter $i: $files3 creates with missing pool" + f3=$(($f3 + $files3)) echo done - time1=$(echo "scale=2; $t1 / $i" | bc) - echo Avg time taken for $numfiles creates without pool: $time1 - time2=$(echo "scale=2; $t2 / $i" | bc) - echo Avg time taken for $numfiles creates with pool: $time2 - time3=$(echo "scale=2; $t3 / $i" | bc) - echo Avg time taken for $numfiles creates with missing pool: $time3 + echo Avg files created in $numsec seconds without pool: $((files1 / iter)) + echo Avg files created in $numsec seconds with pool: $((files2 / iter)) + echo Avg files created in $numsec seconds missing pool: $((files3 / iter)) # Set this high until we establish a baseline for what the degradation # is / should be - max=15 - diff=$(echo "scale=2; ($time2 - $time1) * 100 / $time1" | bc) - echo "No pool to wide pool: $diff %." - deg=$(echo "scale=2; $diff > $max" | bc) - [ "$deg" == "1" ] && error_ignore 23408 "Degradation with wide pool is $diff % (> $max %)" + max=30 + diff=$((($files1 - $files2) * 100 / $files1)) + echo "No pool / wide pool: $diff %." + [ $diff -gt $max ] && + error_ignore 23408 "Degradation with wide pool is $diff% > $max%" max=30 - diff=$(echo "scale=2; ($time3 - $time1) * 100 / $time1" | bc) - echo "No pool to missing pool: $diff %." - deg=$(echo "scale=2; $diff > $max" | bc) - [ "$deg" == "1" ] && error_ignore 23408 "Degradation with missing pool is $diff % (> $max %)" + diff=$((($files1 - $files3) * 100 / $files1)) + echo "No pool / missing pool: $diff %." + [ $diff -gt $max ] && + error_ignore 23408 "Degradation with wide pool is $diff% > $max%" return 0 } run_test 18 "File create in a directory which references a deleted pool" test_19() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local numfiles=12 local dir1=$POOL_ROOT/dir1 @@ -1010,14 +1049,14 @@ test_19() { add_pool $POOL $TGT_HALF "$TGT_UUID2" create_dir $dir1 $POOL - createmany -o $dir1/${tfile} $numfiles || \ + createmany -o $dir1/${tfile} $numfiles || error "createmany $dir1/${tfile} failed!" for file in $dir1/*; do check_file_in_pool $file $POOL done mkdir -p $dir2 - createmany -o $dir2/${tfile} $numfiles || \ + createmany -o $dir2/${tfile} $numfiles || error "createmany $dir2/${tfile} failed!" for file in $dir2/*; do check_file_not_in_pool $file $POOL @@ -1031,6 +1070,7 @@ test_19() { run_test 19 "Pools should not come into play when not specified" test_20() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} local numfiles=12 local dir1=$POOL_ROOT/dir1 @@ -1044,9 +1084,9 @@ test_20() { add_pool $POOL $TGT_HALF "$TGT_UUID2" - local start=$(printf %04x $((TGT_FIRST+1))) - TGT=$(for i in $(seq 0x$start 2 0x$TGT_MAX); \ - do printf "$FSNAME-OST%04x_UUID " $i; done) + local start=$(printf %04x $((TGT_FIRST + 1))) + TGT=$(for i in $(seq 0x$start 2 0x$TGT_MAX); do \ + printf "$FSNAME-OST%04x_UUID " $i; done) add_pool $POOL2 "$FSNAME-OST[$start-$TGT_MAX/2]" "$TGT" create_dir $dir1 $POOL @@ -1079,8 +1119,9 @@ test_20() { run_test 20 "Different pools in a directory hierarchy." test_21() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} - [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return + [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return local numfiles=12 local i=0 @@ -1108,19 +1149,21 @@ add_loop() { echo loop for $pool - for c in $(seq 1 10); - do + for c in $(seq 1 10); do echo "Pool $pool, iteration $c" - do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool OST[$TGT_FIRST-$TGT_MAX/$step] 2>/dev/null - local TGT_SECOND=$(printf %04x $(($TGT_FIRST+$step))) + do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool \ + OST[$TGT_FIRST-$TGT_MAX/$step] 2>/dev/null + local TGT_SECOND=$(printf %04x $((TGT_FIRST + $step))) if [ $((16#$TGT_SECOND)) -le $((16#$TGT_MAX)) ]; then - do_facet $SINGLEMDS lctl pool_remove $FSNAME.$pool OST[$TGT_SECOND-$TGT_MAX/$step] + do_facet $SINGLEMDS lctl pool_remove $FSNAME.$pool \ + OST[$TGT_SECOND-$TGT_MAX/$step] fi done echo loop for $pool complete } test_22() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return @@ -1135,8 +1178,8 @@ test_22() { add_loop $POOL2 2 & sleep 5 create_dir $POOL_ROOT $POOL - createmany -o $POOL_ROOT/${tfile} $numfiles || \ - error "createmany $POOL_ROOT/${tfile} failed!" + createmany -o $POOL_ROOT/${tfile} $numfiles || + error "createmany $POOL_ROOT/${tfile} failed!" wait destroy_pool $POOL @@ -1146,30 +1189,28 @@ test_22() { } run_test 22 "Simultaneous manipulation of a pool" -test_23() { +test_23a() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} - [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return + [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return mkdir -p $POOL_ROOT - check_runas_id $TSTID $TSTID $RUNAS || { + check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS || { skip_env "User $RUNAS_ID does not exist - skipping" return 0 } local i=0 local TGT - local BLK_SZ=1024 - local BUNIT_SZ=1024 # min block quota unit(kB) - local LOVNAME=`lctl get_param -n llite.*.lov.common_name | tail -n 1` - local OSTCOUNT=`lctl get_param -n lov.$LOVNAME.numobd` - local LIMIT + local BUNIT_SZ=1024 # min block quota unit(kB) + local LIMIT=$((BUNIT_SZ * (OSTCOUNT + 1))) local dir=$POOL_ROOT/dir local file="$dir/$tfile-quota" create_pool_nofail $POOL - local TGT=$(for i in $(seq 0x$TGT_FIRST 3 0x$TGT_MAX); \ - do printf "$FSNAME-OST%04x_UUID " $i; done) + local TGT=$(for i in $(seq 0x$TGT_FIRST 3 0x$TGT_MAX); do \ + printf "$FSNAME-OST%04x_UUID " $i; done) add_pool $POOL "$FSNAME-OST[$TGT_FIRST-$TGT_MAX/3]" "$TGT" create_dir $dir $POOL @@ -1183,28 +1224,68 @@ test_23() { sleep 5 fi - LIMIT=$((BUNIT_SZ * (OSTCOUNT + 1))) - $LFS setquota -u $TSTUSR -b $LIMIT -B $LIMIT $dir + $LFS setquota -u $RUNAS_ID -b $LIMIT -B $LIMIT $dir sleep 3 - $LFS quota -v -u $TSTUSR $dir + $LFS quota -v -u $RUNAS_ID $dir $LFS setstripe $file -c 1 -p $POOL - chown $TSTUSR.$TSTUSR $file + chown $RUNAS_ID.$RUNAS_GID $file ls -l $file - type runas - LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BLK_SZ count=$((BUNIT_SZ*2)) || true - $LFS quota -v -u $TSTUSR $dir - cancel_lru_locks osc - stat=$(LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BLK_SZ count=$BUNIT_SZ seek=$((BUNIT_SZ*2)) 2>&1) - RC=$? - echo $stat - [[ $RC -eq 0 ]] && error "dd did not fail with EDQUOT." + # This does two "dd" runs to ensure that the quota failure is returned + # to userspace when we check. The first "dd" might otherwise complete + # without error if it is only writing into cache. + stat=$(LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BUNIT_SZ \ + count=$((BUNIT_SZ*2)) 2>&1) echo $stat | grep "Disk quota exceeded" > /dev/null - [[ $? -eq 1 ]] && error "dd did not fail with EDQUOT." - $LFS quota -v -u $TSTUSR $dir + if [ $? -eq 0 ]; then + $LFS quota -v -u $RUNAS_ID $dir + cancel_lru_locks osc + stat=$(LOCALE=C $RUNAS dd if=/dev/zero of=$file bs=$BUNIT_SZ \ + count=$BUNIT_SZ seek=$((BUNIT_SZ*2)) 2>&1) + RC=$? + echo $stat + [[ $RC -eq 0 ]] && error "second dd did not fail." + echo $stat | grep "Disk quota exceeded" > /dev/null + [[ $? -eq 1 ]] && error "second dd did not fail with EDQUOT." + else + log "first dd failed with EDQUOT." + fi + $LFS quota -v -u $RUNAS_ID $dir +} +run_test 23a "OST pools and quota" + +test_23b() { + set_cleanup_trap + local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} + [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return 0 + + mkdir -p $POOL_ROOT + check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS || { + skip_env "User $RUNAS_ID does not exist - skipping" + return 0 + } + + local i=0 + local TGT + local dir=$POOL_ROOT/dir + local file="$dir/$tfile-quota" + + create_pool_nofail $POOL + + local TGT=$(for i in $(seq 0x$TGT_FIRST 3 0x$TGT_MAX); do \ + printf "$FSNAME-OST%04x_UUID " $i; done) + add_pool $POOL "$FSNAME-OST[$TGT_FIRST-$TGT_MAX/3]" "$TGT" + create_dir $dir $POOL + + local maxfree=$((1024 * 1024 * 30)) # 30G + local AVAIL=$(lfs_df -p $POOL $dir | awk '/summary/ { print $4 }') + [ $AVAIL -gt $maxfree ] && + skip_env "Filesystem space $AVAIL is larger than $maxfree limit" && + return 0 + log "OSTCOUNT=$OSTCOUNT, OSTSIZE=$OSTSIZE" + log "MAXFREE=$MAXFREE, AVAIL=$AVAIL, SLOW=$SLOW" - echo "second run" # XXX remove the interoperability code once we drop the old server # (< 2.3.50) support. if [ $(lustre_version_code mds) -lt $(version_code 2.3.50) ]; then @@ -1213,39 +1294,54 @@ test_23() { do_facet mgs $LCTL conf_param $FSNAME.quota.ost=none sleep 5 fi - # $LFS setquota -u $TSTUSR -b $LIMIT -B $LIMIT $dir - chown $TSTUSR.$TSTUSR $dir + + chown $RUNAS_ID.$RUNAS_ID $dir i=0 - RC=0 - while [ $RC -eq 0 ]; - do - i=$((i+1)) - stat=$(LOCALE=C $RUNAS2 dd if=/dev/zero of=${file}$i bs=1M \ - count=$((LIMIT*LIMIT)) 2>&1) - RC=$? - if [ $RC -eq 1 ]; then - echo $stat - echo $stat | grep "Disk quota exceeded" - [[ $? -eq 0 ]] && error "dd failed with EDQUOT" - - echo $stat | grep "No space left on device" - [[ $? -ne 0 ]] && error "dd did not fail with ENOSPC; " \ - "failed with $stat" - fi + local RC=0 + local TOTAL=0 # KB + local stime=$(date +%s) + local stat + local etime + local elapsed + local maxtime=300 # minimum speed: 5GB / 300sec ~= 17MB/s + while [ $RC -eq 0 ]; do + i=$((i + 1)) + stat=$(LOCALE=C $RUNAS2 dd if=/dev/zero of=${file}$i bs=1M \ + count=$((5 * 1024)) 2>&1) + RC=$? + TOTAL=$((TOTAL + 1024 * 1024 * 5)) + echo "[$i iteration] $stat" + echo "total written: $TOTAL" + + etime=$(date +%s) + elapsed=$((etime - stime)) + echo "stime=$stime, etime=$etime, elapsed=$elapsed" + + if [ $RC -eq 1 ]; then + echo $stat | grep -q "Disk quota exceeded" + [[ $? -eq 0 ]] && error "dd failed with EDQUOT with quota off" + + echo $stat | grep -q "No space left on device" + [[ $? -ne 0 ]] && error "dd did not fail with ENOSPC" + elif [ $TOTAL -gt $AVAIL ]; then + error "dd didn't fail with ENOSPC ($TOTAL > $AVAIL)" + elif [ $i -eq 1 -a $elapsed -gt $maxtime ]; then + log "The first 5G write used $elapsed (> $maxtime) " \ + "seconds, terminated" + RC=1 + fi done df -h rm -rf $POOL_ROOT - destroy_pool $POOL - - return 0 } -run_test 23 "OST pools and quota" +run_test 23b "OST pools and OOS" test_24() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} - [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return + [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return local numfiles=10 local i=0 @@ -1253,7 +1349,6 @@ test_24() { local dir local res - create_pool_nofail $POOL add_pool $POOL $TGT_ALL "$TGT_UUID" @@ -1261,59 +1356,58 @@ test_24() { create_dir $POOL_ROOT/dir1 $POOL $OSTCOUNT mkdir $POOL_ROOT/dir2 - $SETSTRIPE $POOL_ROOT/dir2 -p $POOL -s 65536 -i 0 -c 1 || \ + $SETSTRIPE $POOL_ROOT/dir2 -p $POOL -s 65536 -i 0 -c 1 || error "$SETSTRIPE $POOL_ROOT/dir2 failed" mkdir $POOL_ROOT/dir3 - $SETSTRIPE $POOL_ROOT/dir3 -s 65536 -i 0 -c 1 || \ + $SETSTRIPE $POOL_ROOT/dir3 -s 65536 -i 0 -c 1 || error "$SETSTRIPE $POOL_ROOT/dir3 failed" mkdir $POOL_ROOT/dir4 - for i in $(seq 1 4); - do - dir=${POOL_ROOT}/dir${i} - local pool - local pool1 - local count - local count1 - local index - local size - local size1 - - createmany -o $dir/${tfile} $numfiles || \ - error "createmany $dir/${tfile} failed!" - res=$($GETSTRIPE -v $dir | grep "^stripe_count:") - if [ $? -ne 0 ]; then - res=$($GETSTRIPE -v $dir | grep "^(Default) ") - pool=$(cut -f 9 -d ' ' <<< $res) - index=$(cut -f 7 -d ' ' <<< $res) - size=$(cut -f 5 -d ' ' <<< $res) - count=$(cut -f 3 -d ' ' <<< $res) - else - pool=$(cut -f 8 -d ' ' <<< $res) - index=$(cut -f 6 -d ' ' <<< $res) - size=$(cut -f 4 -d ' ' <<< $res) - count=$(cut -f 2 -d ' ' <<< $res) - fi - - for file in $dir/*; do - if [ "$pool" != "" ]; then - check_file_in_pool $file $pool - fi - pool1=$($GETSTRIPE -v $file | grep "^pool:" | \ - tr -d '[:blank:]' | cut -f 2 -d ':') - count1=$($GETSTRIPE -v $file | grep "^lmm_stripe_count:" | \ - tr -d '[:blank:]' | cut -f 2 -d ':') - size1=$($GETSTRIPE -v $file | grep "^lmm_stripe_size:" | \ - tr -d '[:blank:]' | cut -f 2 -d ':') - [[ "$pool" != "$pool1" ]] && \ - error "Pool name ($pool) not inherited in $file($pool1)" - [[ "$count" != "$count1" ]] && \ - error "Stripe count ($count) not inherited in $file ($count1)" - [[ "$size" != "$size1" ]] && [[ "$size" != "0" ]] && \ - error "Stripe size ($size) not inherited in $file ($size1)" - done + for i in 1 2 3 4; do + dir=${POOL_ROOT}/dir${i} + local pool + local pool1 + local count + local count1 + local index + local size + local size1 + + createmany -o $dir/${tfile} $numfiles || + error "createmany $dir/${tfile} failed!" + res=$($GETSTRIPE -v $dir | grep "^stripe_count:") + if [ $? -ne 0 ]; then + res=$($GETSTRIPE -v $dir | grep "^(Default) ") + pool=$(cut -f 9 -d ' ' <<< $res) + index=$(cut -f 7 -d ' ' <<< $res) + size=$(cut -f 5 -d ' ' <<< $res) + count=$(cut -f 3 -d ' ' <<< $res) + else + pool=$(cut -f 8 -d ' ' <<< $res) + index=$(cut -f 6 -d ' ' <<< $res) + size=$(cut -f 4 -d ' ' <<< $res) + count=$(cut -f 2 -d ' ' <<< $res) + fi + + for file in $dir/*; do + if [ "$pool" != "" ]; then + check_file_in_pool $file $pool + fi + pool1=$($GETSTRIPE -v $file | grep "^pool:" | + tr -d '[:blank:]' | cut -f 2 -d ':') + count1=$($GETSTRIPE -v $file | grep "^lmm_stripe_count:" | + tr -d '[:blank:]' | cut -f 2 -d ':') + size1=$($GETSTRIPE -v $file | grep "^lmm_stripe_size:" | + tr -d '[:blank:]' | cut -f 2 -d ':') + [[ "$pool" != "$pool1" ]] && + error "Pool name ($pool) not inherited in $file($pool1)" + [[ "$count" != "$count1" ]] && + error "Stripe count ($count) not inherited in $file ($count1)" + [[ "$size" != "$size1" ]] && [[ "$size" != "0" ]] && + error "Stripe size ($size) not inherited in $file ($size1)" + done done rm -rf $POOL_ROOT @@ -1324,19 +1418,20 @@ test_24() { run_test 24 "Independence of pool from other setstripe parameters" test_25() { + set_cleanup_trap local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir} mkdir -p $POOL_ROOT for i in $(seq 10); do - create_pool_nofail pool$i - do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool$i OST0000; sync" - wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool$i | \ - sort -u | tr '\n' ' ' " "$FSNAME-OST0000_UUID " || \ - error "pool_add failed: $1; $2" + create_pool_nofail $POOL$i + do_facet $SINGLEMDS "lctl pool_add $FSNAME.$POOL$i OST0000; sync" + wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$POOL$i | + sort -u | tr '\n' ' ' " "$FSNAME-OST0000_UUID " >/dev/null || + error "pool_add failed: $1; $2" stop $SINGLEMDS || return 1 - start $SINGLEMDS $MDSDEV $MDS_MOUNT_OPTS || \ + start $SINGLEMDS $MDSDEV $MDS_MOUNT_OPTS || { error "Failed to start $SINGLEMDS after stopping" && break; } wait_osc_import_state mds ost FULL clients_up @@ -1349,9 +1444,9 @@ test_25() { STR=$($GETSTRIPE $POOL_ROOT/$tfile | grep 0x | cut -f2 | tr -d " ") rm $POOL_ROOT/$tfile if [[ "$STR" == "0" ]]; then - echo "Creating a file in pool$i" - create_file $POOL_ROOT/file$i pool$i || break - check_file_in_pool $POOL_ROOT/file$i pool$i || break + echo "Creating a file in $POOL$i" + create_file $POOL_ROOT/file$i $POOL$i || break + check_file_in_pool $POOL_ROOT/file$i $POOL$i || break else echo "OST 0 seems to be unavailable. Try later." fi @@ -1359,10 +1454,10 @@ test_25() { rm -rf $POOL_ROOT for i in $(seq 10); do - destroy_pool pool$i + destroy_pool $POOL$i done } -run_test 25 "Create new pool and restart MDS =======================" +run_test 25 "Create new pool and restart MDS" cd $ORIG_PWD diff --git a/lustre/tests/sanity-benchmark.sh b/lustre/tests/sanity-benchmark.sh index 2ea5b3d..9444548 100644 --- a/lustre/tests/sanity-benchmark.sh +++ b/lustre/tests/sanity-benchmark.sh @@ -236,7 +236,7 @@ space_check () { local stripe=$($LFS getstripe -c $testdir) # if stripe_count = 1 the size should be less than min ost size, bug 24294 - local space=$($LFS df $testdir | grep "filesystem summary:" | awk '{print $3}') + local space=$(lfs_df $testdir | grep "summary" | awk '{print $2}') [ $stripe -eq 1 ] && space=$(min_ost_size) local size=$(pios_file_size) diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh index 61d3ad0..ae1d317 100644 --- a/lustre/tests/sanity-quota.sh +++ b/lustre/tests/sanity-quota.sh @@ -376,8 +376,7 @@ test_0() { local MB=100 # 100M [ "$SLOW" = "no" ] && MB=10 - local free_space=$(lfs df | grep "filesystem summary" | \ - awk '{print $5}') + local free_space=$(lfs_df | grep "summary" | awk '{print $5}') [ $free_space -le $((MB * 1024)) ] && skip "not enough space ${free_space} KB, " \ "required $((MB * 1024)) KB" && return @@ -481,8 +480,7 @@ test_2() { [ "$SLOW" = "no" ] && LIMIT=1024 # 1k inodes - local FREE_INODES=$(lfs df -i | grep "filesystem summary" | \ - awk '{print $5}') + local FREE_INODES=$(lfs_df -i | grep "summary" | awk '{print $5}') [ $FREE_INODES -lt $LIMIT ] && skip "not enough free inodes $FREE_INODES required $LIMIT" && return @@ -503,7 +501,7 @@ test_2() { [ $USED -ne 0 ] && error "Used inodes($USED) for user $TSTUSR isn't 0." log "Create $LIMIT files ..." - $RUNAS createmany -m ${TESTFILE} $LIMIT || \ + $RUNAS createmany -m ${TESTFILE} $LIMIT || quota_error u $TSTUSR "user create failure, but expect success" log "Create out of file quota ..." $RUNAS touch ${TESTFILE}_xxx && \ diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh index 1105570..64db399 100644 --- a/lustre/tests/sanityn.sh +++ b/lustre/tests/sanityn.sh @@ -473,7 +473,7 @@ run_test 24a "lfs df [-ih] [path] test =========================" test_24b() { touch $DIR1/$tfile - fsnum=`lfs df | grep -c "filesystem summary:"` + fsnum=$(lfs_df | grep -c "summary") [ $fsnum -eq 2 ] || error "lfs df shows $fsnum != 2 filesystems." } run_test 24b "lfs df should show both filesystems ===============" diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 6071cf4..02ffed1 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -634,6 +634,13 @@ restore_quota_old() { quota_save_version $old_QUOTA_TYPE } +# Handle the case when there is a space in the lfs df +# "filesystem summary" line the same as when there is no space. +# This will allow fixing the "lfs df" summary line in the future. +lfs_df() { + $LFS df $* | sed -e 's/filesystem /filesystem_/' +} + # XXX This function is kept for interoperability with old server (< 2.3.50), # it should be removed whenever we drop the interoperability for such # server. @@ -656,11 +663,11 @@ setup_quota_old() { local quota_usrs=$QUOTA_USERS # get_filesystem_size - local disksz=$(lfs df $mntpt | grep "filesystem summary:" | awk '{print $3}') + local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}') local blk_soft=$((disksz + 1024)) local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over - local Inodes=$(lfs df -i $mntpt | grep "filesystem summary:" | awk '{print $3}') + local Inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}') local i_soft=$Inodes local i_hard=$((i_soft + i_soft / 20)) @@ -740,12 +747,12 @@ setup_quota() { local quota_usrs=$QUOTA_USERS # get_filesystem_size - local disksz=$(lfs df $mntpt | grep "filesystem summary:" | + local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $3}') local blk_soft=$((disksz + 1024)) local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over - local inodes=$(lfs df -i $mntpt | grep "filesystem summary:" | + local inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $3}') local i_soft=$inodes local i_hard=$((i_soft + i_soft / 20)) @@ -1309,7 +1316,8 @@ wait_update () { while [ true ]; do RESULT=$(do_node $node "$TEST") if [ "$RESULT" == "$FINAL" ]; then - echo "Updated after $WAIT sec: wanted '$FINAL' got '$RESULT'" + [ -z "$RESULT" -o $WAIT -le $sleep ] || + echo "Updated after ${WAIT}s: wanted '$FINAL' got '$RESULT'" return 0 fi [ $WAIT -ge $MAX ] && break @@ -1317,7 +1325,7 @@ wait_update () { WAIT=$((WAIT + sleep)) sleep $sleep done - echo "Update not seen after $MAX sec: wanted '$FINAL' got '$RESULT'" + echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'" return 3 } -- 1.8.3.1