X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fost-pools.sh;h=062924a58a3448835bafcd70fe607459e04c96a1;hb=81566e1dfdc1950bc739e50add54b8727e3accc1;hp=ea02f0b1d0f3687747fc74b7342180f27248623c;hpb=9dd947a09c39361088f1ec4d993a8c5ff7102423;p=fs%2Flustre-release.git

diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh
index ea02f0b..062924a 100644
--- a/lustre/tests/ost-pools.sh
+++ b/lustre/tests/ost-pools.sh
@@ -32,8 +32,6 @@ check_and_setup_lustre
 DIR=${DIR:-$MOUNT}
 assert_DIR
 
-POOLSLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
-[ "$POOLSLOG" ] && rm -f $POOLSLOG || true
 build_test_filter
 
 FAIL_ON_ERROR=${FAIL_ON_ERROR:-true}
@@ -56,11 +54,11 @@ NON_EXISTANT_POOL=nonexistantpool
 NON_EXISTANT_FS=nonexistantfs
 INVALID_POOL=some_invalid_pool_name
 TGT_COUNT=$OSTCOUNT
-TGT_FIRST=0
-TGT_MAX=$((TGT_COUNT-1))
+TGT_FIRST=$(printf %04x 0)
+TGT_MAX=$(printf %04x $((TGT_COUNT-1)))
 TGT_STEP=1
-TGT_LIST=$(seq $TGT_FIRST $TGT_STEP $TGT_MAX)
-TGT_LIST2=$(seq $TGT_FIRST 2 $TGT_MAX)
+TGT_LIST=$(seq 0x$TGT_FIRST $TGT_STEP 0x$TGT_MAX)
+TGT_LIST2=$(seq 0x$TGT_FIRST 2 0x$TGT_MAX)
 
 TGT_ALL="$FSNAME-OST[$TGT_FIRST-$TGT_MAX/1]"
 TGT_HALF="$FSNAME-OST[$TGT_FIRST-$TGT_MAX/2]"
@@ -148,10 +146,15 @@ check_file_in_osts() {
         return 0
 }
 
+file_pool() {
+    $GETSTRIPE -v $1 | grep "^lmm_pool:" | tr -d "[:blank:]" | cut -f 2 -d ':'
+}
+
 check_file_not_in_pool() {
     local file=$1
     local pool=$2
-    local res=$($GETSTRIPE -v $file | grep "^pool:" | tr -d "[:blank:]" | cut -f 2 -d ':')
+    local res=$(file_pool $file)
+
     if [[ "$res" == "$pool" ]]; then
         error "File $file is in pool: $res"
         return 1
@@ -224,16 +227,19 @@ ost_pools_init() {
     cleanup_tests
 }
 
+set_cleanup_trap() {
+    trap "cleanup_pools $FSNAME" EXIT
+}
 
 # Initialization
 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
 ost_pools_init
 
-trap "cleanup_pools $FSNAME" EXIT
 
 # Tests for new commands added
 test_1() {
+    set_cleanup_trap
     echo "Creating a pool with a 1 character pool name"
     create_pool_nofail p
 
@@ -249,9 +255,9 @@ test_1() {
     create_pool_fail p1234567891234567
 
     echo "Creating a pool with a 1000 character pool name; should fail"
-    NAME="p"
-    for i in `seq 1 999`; do NAME=${NAME}"o"; done
-    create_pool_fail $NAME
+    local pool_name="p"
+    for ((i=1;i<=999;i++)); do pool_name=${pool_name}"o"; done
+    create_pool_fail $pool_name
 
     echo "pool_new should fail if fs-name or poolname are missing."
     do_facet $SINGLEMDS lctl pool_new .pool1 2>/dev/null
@@ -283,6 +289,7 @@ test_1() {
 run_test 1 "Test lctl pool_new  ========================================="
 
 test_2a() {
+    set_cleanup_trap
     destroy_pool $POOL
 
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null
@@ -293,6 +300,7 @@ test_2a() {
 run_test 2a "pool_add: non-existant pool"
 
 test_2b() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_add $FSNAME.p1234567891234567890 \
         $FSNAME-OST0000 2>/dev/null
     [[ $? -ne 0 ]] || \
@@ -302,6 +310,7 @@ run_test 2b "pool_add: Invalid pool name"
 
 # Testing various combinations of OST name list
 test_2c() {
+    set_cleanup_trap
     local TGT
     local RC
 
@@ -318,23 +327,23 @@ test_2c() {
     do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL OST0000
     drain_pool $POOL
 
-    # 2. lustre-OST0000
+    # 2. $FSNAME-OST0000
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000
     RC=$?; [[ $RC -eq 0 ]] || \
         error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000: $RC"
     do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000
     drain_pool $POOL
 
-    # 3. lustre-OST0000_UUID
+    # 3. $FSNAME-OST0000_UUID
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000_UUID
     RC=$?; [[ $RC -eq 0 ]] || \
         error "pool_add failed. $FSNAME $POOL $FSNAME-OST0000_UUID: $RC"
     do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $FSNAME-OST0000_UUID
     drain_pool $POOL
 
-    # 4. lustre-OST[0,1,2,3,]
+    # 4. $FSNAME-OST[0,1,2,3,]
     TGT="$FSNAME-OST["
-    for i in $TGT_LIST; do TGT=${TGT}$(printf "$i," $i); done
+    for i in $TGT_LIST; do TGT=${TGT}$(printf "%04x," $i); done
     TGT="${TGT}]"
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT
     [[ $? -eq 0 ]] || \
@@ -342,7 +351,7 @@ test_2c() {
     do_facet $SINGLEMDS lctl pool_remove $FSNAME.$POOL $TGT
     drain_pool $POOL
 
-    # 5. lustre-OST[0-5/1]
+    # 5. $FSNAME-OST[0-5/1]
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $TGT_ALL
     RC=$?; [[ $RC -eq 0 ]] || \
         error "pool_add failed. $FSNAME $POOL" "$TGT_ALL $RC"
@@ -356,6 +365,7 @@ test_2c() {
 run_test 2c "pool_add: OST index combinations ==========================="
 
 test_2d() {
+    set_cleanup_trap
     local TGT
     local RC
 
@@ -375,6 +385,7 @@ test_2d() {
 run_test 2d "pool_add: OSTs that don't exist should be rejected ========"
 
 test_2e() {
+    set_cleanup_trap
     local TGT
     local RC
     local RESULT
@@ -405,6 +416,7 @@ test_2e() {
 run_test 2e "pool_add: OST already in a pool should be rejected ========"
 
 test_3a() {
+    set_cleanup_trap
     lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         destroy_pool $POOL
@@ -416,6 +428,7 @@ test_3a() {
 run_test 3a "pool_remove: non-existant pool"
 
 test_3b() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_remove ${NON_EXISTANT_FS}.$POOL OST0000 2>/dev/null
     [[ $? -ne 0 ]] || \
         error "pool_remove did not fail even though fsname did not exist."
@@ -423,6 +436,7 @@ test_3b() {
 run_test 3b "pool_remove: non-existant fsname"
 
 test_3c() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_remove $FSNAME.p1234567891234567890 \
         $FSNAME-OST0000 2>/dev/null
     [[ $? -ne 0 ]] || \
@@ -432,6 +446,7 @@ run_test 3c "pool_remove: Invalid pool name"
 
 # Testing various combinations of OST name list
 test_3d() {
+    set_cleanup_trap
     lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         destroy_pool $POOL
@@ -466,6 +481,7 @@ test_3d() {
 run_test 3d "pool_remove: OST index combinations ==========================="
 
 test_4a() {
+    set_cleanup_trap
     lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         destroy_pool $POOL
@@ -477,6 +493,7 @@ test_4a() {
 run_test 4a "pool_destroy: non-existant pool"
 
 test_4b() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_destroy ${NON_EXISTANT_FS}.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         error "pool_destroy did not fail even though the filesystem did not exist."
@@ -484,6 +501,7 @@ test_4b() {
 run_test 4b "pool_destroy: non-existant fs-name"
 
 test_4c() {
+    set_cleanup_trap
     create_pool_nofail $POOL
     add_pool $POOL "OST0000" "$FSNAME-OST0000_UUID "
 
@@ -546,6 +564,7 @@ sub_test_5() {
 }
 
 test_5() {
+    set_cleanup_trap
     # Issue commands from client
     sub_test_5 $LCTL
     sub_test_5 $LFS
@@ -558,6 +577,7 @@ test_5() {
 run_test 5 "lfs/lctl pool_list"
 
 test_6() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local POOL_DIR=$POOL_ROOT/dir_tst
     local POOL_FILE=$POOL_ROOT/file_tst
@@ -602,12 +622,11 @@ test_6() {
     [[ $? -ne 0 ]] || \
         error "$SETSTRIPE with start index outside the pool did not fail."
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
 }
 run_test 6 "getstripe/setstripe"
 
 test_11() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
 
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
@@ -615,7 +634,7 @@ test_11() {
     create_pool_nofail $POOL
     create_pool_nofail $POOL2
 
-    local start=$((TGT_FIRST+1))
+    local start=$(printf %04x $((TGT_FIRST+1)))
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \
         $FSNAME-OST[$start-$TGT_MAX/2]
 
@@ -642,14 +661,12 @@ test_11() {
 
     rm -rf $POOL_ROOT/dir?
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
-
     return 0
 }
 run_test 11 "OSTs in overlapping/multiple pools"
 
 test_12() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
 
     [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
@@ -657,7 +674,7 @@ test_12() {
     create_pool_nofail $POOL
     create_pool_nofail $POOL2
 
-    local start=$((TGT_FIRST+1))
+    local start=$(printf %04x $((TGT_FIRST+1)))
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL2 \
         $FSNAME-OST[$start-$TGT_MAX/2]
 
@@ -687,7 +704,7 @@ test_12() {
     check_dir_in_pool $POOL_ROOT/dir1 $POOL
     check_dir_in_pool $POOL_ROOT/dir2 $POOL2
     check_file_in_osts $POOL_ROOT/file1 "$TGT_LIST2"    
-    check_file_in_osts $POOL_ROOT/file2 "$(seq $start 2 $TGT_MAX)"
+    check_file_in_osts $POOL_ROOT/file2 "$(seq 0x$start 2 0x$TGT_MAX)"
 
     echo Creating some more files
     create_dir $POOL_ROOT/dir3 $POOL
@@ -699,14 +716,12 @@ test_12() {
     check_file_in_pool $POOL_ROOT/file3 $POOL
     check_file_in_pool $POOL_ROOT/file4 $POOL2
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
-
     return 0    
 }
 run_test 12 "OST Pool Membership"
 
 test_13() {
+    set_cleanup_trap
     [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
 
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
@@ -729,7 +744,7 @@ test_13() {
     check_file_in_pool $POOL_ROOT/dir1/file1 $POOL 1
     check_file_in_pool $POOL_ROOT/dir1/file2 $POOL 1
     create_file $POOL_ROOT/dir1/file3 $POOL 1 $((TGT_FIRST+2))
-    check_file_in_osts $POOL_ROOT/dir1/file1 "$TGT_FIRST"
+    check_file_in_osts $POOL_ROOT/dir1/file1 $((16#$TGT_FIRST))
     check_file_in_osts $POOL_ROOT/dir1/file2 "$((TGT_FIRST+1))"
     check_file_in_osts $POOL_ROOT/dir1/file3 "$((TGT_FIRST+2))"
 
@@ -763,13 +778,13 @@ test_13() {
     done
 
     rm -rf create_dir $POOL_ROOT/dir?
-    destroy_pool $POOL
 
     return 0
 }
 run_test 13 "Striping characteristics in a pool"
 
 test_14() {
+    set_cleanup_trap
     [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
 
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
@@ -793,7 +808,7 @@ test_14() {
     while [[ $i -lt $numfiles ]];
     do
         OST=$((OST+2))
-        [[ $OST -gt $TGT_MAX ]] && OST=$TGT_FIRST
+        [[ $OST -gt $((16#$TGT_MAX)) ]] && OST=$TGT_FIRST
 
         # echo "Iteration: $i OST: $OST"
         create_file $POOL_ROOT/dir1/file${i} $POOL 1
@@ -826,13 +841,12 @@ test_14() {
     done
 
     rm -rf $POOL_ROOT
-    destroy_pool $POOL
-    destroy_pool $POOL2
     return 0
 }
 run_test 14 "Round robin and QOS striping within a pool"
 
 test_15() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=100
     local i=0
@@ -842,7 +856,7 @@ test_15() {
       create_pool_nofail pool${i}
 
       local tgt=$(printf "$FSNAME-OST%04x_UUID " $i)
-      add_pool pool${i} "$FSNAME-OST[$i]" "$tgt"
+      add_pool pool${i} "$FSNAME-OST[$(printf %04x $i)]" "$tgt"
       create_dir $POOL_ROOT/dir${i} pool${i}
       createmany -o $POOL_ROOT/dir$i/$tfile $numfiles || \
           error "createmany $POOL_ROOT/dir$i/$tfile failed!"
@@ -854,18 +868,12 @@ test_15() {
       i=$((i+1))
     done
 
-    i=0
-    while [[ $i -lt $OSTCOUNT ]]
-    do
-      destroy_pool pool${i}
-      i=$((i+1))
-    done
-
     return 0
 }
 run_test 15 "One directory per OST/pool"
 
 test_16() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=10
     local i=0
@@ -891,13 +899,12 @@ test_16() {
 
     rm -rf $POOL_ROOT/$tdir
 
-    destroy_pool $POOL
-
     return 0
 }
 run_test 16 "Inheritance of pool properties"
 
 test_17() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=10
     local i=0
@@ -926,72 +933,83 @@ test_17() {
 }
 run_test 17 "Referencing an empty pool"
 
-create_perf_one() {
-    local dir=$1/d
+create_perf() {
+    local cdir=$1/d
     local numfiles=$2
     local time
 
-    mkdir -p $dir
+    mkdir -p $cdir
     sync; sleep 5 # give pending IO a chance to go to disk
-    stat=$(createmany -o $dir/${tfile} $numfiles)
-    rm -rf $dir
+    stat=$(createmany -o $cdir/${tfile} $numfiles)
+    rm -rf $cdir
     sync
     time=$(echo $stat | cut -f 5 -d ' ')
-    echo $stat > /dev/stderr
+    echo $stat >> /dev/stderr
     echo $time
 }
 
-create_perf() {
-    local t1=$(create_perf_one $1 $2)
-    local t2=$(create_perf_one $1 $2)
-    local t3=$(create_perf_one $1 $2)
-    local time=$(echo "scale=2; ( $t1 + $t2 + $t3 ) / 3" | bc)
-
-    echo "$time"
-}
-
 test_18() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
-    local numfiles=10000
-    local i=0
-    local time1
-    local time2
-    local time3
-    local dir=$POOL_ROOT/dir
-    local stat
-    local diff
-
-    time1=$(create_perf $dir $numfiles)
-
-    create_pool_nofail $POOL
-
-    add_pool $POOL $TGT_ALL "$TGT_UUID"
-
-    create_dir $dir $POOL
-    time2=$(create_perf $dir $numfiles)
-
-    destroy_pool $POOL
-
-    time3=$(create_perf $dir $numfiles)
-
-    echo Time taken for $numfiles creates without pools: $time1
-    echo Time taken for $numfiles creates with pools: $time2
-    echo Time taken for $numfiles creates without pools: $time3
-
-    deg=$(echo "scale=2; (($time2 - $time3) * 100 / $time3) > 5" | bc)
-    diff=$(echo "scale=2; ($time2 - $time3) * 100 / $time3" | bc)
+    local numfiles=30000
+    local plaindir=$POOL_ROOT/plaindir
+    local pooldir=$POOL_ROOT/pooldir
+	local t1=0
+	local t2=0
+	local t3=0
+	local diff
+
+    for i in $(seq 1 3);
+    do
+        echo "Create performance, iteration $i, $numfiles files x 3"
+
+		time1=$(create_perf $plaindir $numfiles)
+		echo "iter $i: $numfiles creates without pool: $time1"
+		t1=$(echo "scale=2; $t1 + $time1" | bc)
+
+		create_pool_nofail $POOL > /dev/null
+		add_pool $POOL $TGT_ALL "$TGT_UUID" > /dev/null
+		create_dir $pooldir $POOL
+		time2=$(create_perf $pooldir $numfiles)
+		echo "iter $i: $numfiles creates with pool: $time2"
+		t2=$(echo "scale=2; $t2 + $time2" | bc)
+
+		destroy_pool $POOL > /dev/null
+		time3=$(create_perf $pooldir $numfiles)
+		echo "iter $i: $numfiles creates with missing pool: $time3"
+		t3=$(echo "scale=2; $t3 + $time3" | bc)
+
+		echo
+	done
+
+	time1=$(echo "scale=2; $t1 / $i" | bc)
+	echo Avg time taken for $numfiles creates without pool: $time1
+	time2=$(echo "scale=2; $t2 / $i" | bc)
+	echo Avg time taken for $numfiles creates with pool: $time2
+	time3=$(echo "scale=2; $t3 / $i" | bc)
+	echo Avg time taken for $numfiles creates with missing pool: $time3
+
+	# Set this high until we establish a baseline for what the degradation
+	# is / should be
+	max=30
+    diff=$(echo "scale=2; ($time2 - $time1) * 100 / $time1" | bc)
+    echo  "No pool to wide pool: $diff %."
+    deg=$(echo "scale=2; $diff > $max" | bc)
+    [ "$deg" == "1" ] && error_ignore 23408  "Degradation with wide pool is $diff % (> $max %)"
+
+	max=15
+    diff=$(echo "scale=2; ($time3 - $time1) * 100 / $time1" | bc)
+    echo  "No pool to missing pool: $diff %."
+    deg=$(echo "scale=2; $diff > $max" | bc)
+    [ "$deg" == "1" ] && error_ignore 23408 "Degradation with missing pool is $diff % (> $max %)"
 
-    if [[ "$deg" == "1" ]]; then
-        error "Performance degradation with pools is $diff %."
-    else
-        echo  "Performance degradation with pools is $diff %."
-    fi
     return 0
 }
 run_test 18 "File create in a directory which references a deleted pool"
 
 
 test_19() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=12
     local dir1=$POOL_ROOT/dir1
@@ -1017,13 +1035,13 @@ test_19() {
     done
 
     rm -rf $dir1 $dir2
-    destroy_pool $POOL
 
     return 0
 }
 run_test 19 "Pools should not come into play when not specified"
 
 test_20() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=12
     local dir1=$POOL_ROOT/dir1
@@ -1037,8 +1055,8 @@ test_20() {
 
     add_pool $POOL $TGT_HALF "$TGT_UUID2"
 
-    local start=$((TGT_FIRST+1))
-    TGT=$(for i in `seq $start 2 $TGT_MAX`; \
+    local start=$(printf %04x $((TGT_FIRST+1)))
+    TGT=$(for i in $(seq 0x$start 2 0x$TGT_MAX); \
         do printf "$FSNAME-OST%04x_UUID " $i; done)
     add_pool $POOL2 "$FSNAME-OST[$start-$TGT_MAX/2]" "$TGT"
 
@@ -1064,14 +1082,13 @@ test_20() {
     check_file_not_in_pool $dir2/file4 $POOL2
 
     rm -rf $dir1
-    destroy_pool $POOL
-    destroy_pool $POOL2
 
     return 0
 }
 run_test 20 "Different pools in a directory hierarchy."
 
 test_21() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
 
@@ -1089,7 +1106,6 @@ test_21() {
     check_file_in_pool $dir/file1 $POOL
 
     rm -rf $dir
-    destroy_pool $POOL
 
     return 0
 }
@@ -1105,8 +1121,8 @@ add_loop() {
     do
         echo "Pool $pool, iteration $c"
 	do_facet $SINGLEMDS lctl pool_add $FSNAME.$pool OST[$TGT_FIRST-$TGT_MAX/$step] 2>/dev/null
-	local TGT_SECOND=$(($TGT_FIRST+$step))
-	if [ "$TGT_SECOND" -le "$TGT_MAX" ]; then
+	local TGT_SECOND=$(printf %04x $((TGT_FIRST+$step)))
+	if [ $((16#$TGT_SECOND)) -le $((16#$TGT_MAX)) ]; then
 	    do_facet $SINGLEMDS lctl pool_remove $FSNAME.$pool OST[$TGT_SECOND-$TGT_MAX/$step]
 	fi
     done
@@ -1114,6 +1130,7 @@ add_loop() {
 }
 
 test_22() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return
 
@@ -1132,14 +1149,12 @@ test_22() {
           error "createmany $POOL_ROOT/${tfile} failed!"
     wait
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
-
     return 0
 }
 run_test 22 "Simultaneous manipulation of a pool"
 
 test_23() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
 
@@ -1161,7 +1176,7 @@ test_23() {
 
     create_pool_nofail $POOL
 
-    local TGT=$(for i in `seq $TGT_FIRST 3 $TGT_MAX`; \
+    local TGT=$(for i in $(seq 0x$TGT_FIRST 3 0x$TGT_MAX); \
         do printf "$FSNAME-OST%04x_UUID " $i; done)
     add_pool $POOL "$FSNAME-OST[$TGT_FIRST-$TGT_MAX/3]" "$TGT"
     create_dir $dir $POOL
@@ -1215,13 +1230,13 @@ test_23() {
     df -h
 
     rm -rf $POOL_ROOT
-    destroy_pool $POOL
 
     return 0
 }
 run_test 23 "OST pools and quota"
 
 test_24() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
 
@@ -1279,8 +1294,7 @@ test_24() {
           if [ "$pool" != "" ]; then
               check_file_in_pool $file $pool
           fi
-          pool1=$($GETSTRIPE -v $file | grep "^pool:" |\
-              tr -d '[:blank:]' | cut -f 2 -d ':')
+          pool1=$(file_pool $file)
           count1=$($GETSTRIPE -v $file | grep "^lmm_stripe_count:" |\
               tr -d '[:blank:]' | cut -f 2 -d ':')
           size1=$($GETSTRIPE -v $file | grep "^lmm_stripe_size:" |\
@@ -1295,13 +1309,13 @@ test_24() {
     done
 
     rm -rf $POOL_ROOT
-    destroy_pool $POOL
 
     return 0
 }
 run_test 24 "Independence of pool from other setstripe parameters"
 
 test_25() {
+    set_cleanup_trap
     local dev=$(mdsdevname ${SINGLEMDS//mds/})
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
 
@@ -1309,30 +1323,79 @@ test_25() {
 
     for i in $(seq 10); do
         create_pool_nofail pool$i
-	do_facet $SINGLEMDS lctl pool_add $FSNAME.pool$i OST0000
+        do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool$i OST0000; sync"
+        wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool$i | \
+            sort -u | tr '\n' ' ' " "$FSNAME-OST0000_UUID " || \
+            error "pool_add failed: $1; $2"
+
         stop $SINGLEMDS || return 1
         start $SINGLEMDS ${dev} $MDS_MOUNT_OPTS  || \
             { error "Failed to start $SINGLEMDS after stopping" && break; }
         wait_osc_import_state mds ost FULL
         clients_up
 
+        wait_mds_ost_sync 
         # Veriy that the pool got created and is usable
-        echo "Creating a file in pool$i"
-        create_file $POOL_ROOT/file$i pool$i || break
-        check_file_in_pool $POOL_ROOT/file$i pool$i || break
+        df $POOL_ROOT > /dev/null
+        sleep 5
+        # Make sure OST0 can be striped on
+        $SETSTRIPE -o 0 -c 1 $POOL_ROOT/$tfile
+        STR=$($GETSTRIPE $POOL_ROOT/$tfile | grep 0x | cut -f2 | tr -d " ")
+        rm $POOL_ROOT/$tfile
+        if [[ "$STR" == "0" ]]; then
+                echo "Creating a file in pool$i"
+                create_file $POOL_ROOT/file$i pool$i || break
+                check_file_in_pool $POOL_ROOT/file$i pool$i || break
+        else
+                echo "OST 0 seems to be unavailable.  Try later."
+        fi
     done
 
     rm -rf $POOL_ROOT
-    for i in $(seq 10); do
-        destroy_pool pool$i
-    done
 }
 run_test 25 "Create new pool and restart MDS ======================="
 
-log "cleanup: ======================================================"
+test_26() {
+    local OSTCOUNT=`lctl get_param -n lov.$LOVNAME.*clilov*.numobd`
+    [[ $OSTCOUNT -le 2 ]] && skip_env "Need at least 3 OSTs" && return
+    set_cleanup_trap
+    local dev=$(mdsdevname ${SINGLEMDS//mds/})
+    local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
+
+    mkdir -p $POOL_ROOT
+
+    create_pool_nofail $POOL 
+ 
+    do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0000; sync"
+    wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
+    sort -u | grep $FSNAME-OST0000_UUID " "$FSNAME-OST0000_UUID" || \
+    error "pool_add failed: $1; $2"
+  
+    do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0002; sync"
+    wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
+    sort -u | grep $FSNAME-OST0002_UUID" "$FSNAME-OST0002_UUID" || \
+    error "pool_add failed: $1; $2"
+  
+    # Veriy that the pool got created and is usable
+    df $POOL_ROOT
+    echo "Creating files in $POOL"
+
+    for ((i=0;i<10;i++)); do
+        #OBD_FAIL_MDS_OSC_CREATE_FAIL     0x147  
+        #Fail OST0000 to make sure the objects create on the other ost in the pool 
+        do_facet $SINGLEMDS lctl set_param fail_loc=0x147
+        do_facet $SINGLEMDS lctl set_param fail_val=0
+        create_file $POOL_ROOT/file$i $POOL 1 -1 || break
+        do_facet $SINGLEMDS lctl set_param fail_loc=0
+        check_file_in_pool $POOL_ROOT/file$i $POOL || break
+    done
+    rm -rf $POOL_ROOT
+}
+run_test 26 "Choose other OSTs in the pool first in the creation remedy"
+
 cd $ORIG_PWD
+
+complete $(basename $0) $SECONDS
 cleanup_pools $FSNAME
 check_and_cleanup_lustre
-echo '=========================== finished ==============================='
-[ -f "$POOLSLOG" ] && cat $POOLSLOG && grep -q FAIL $POOLSLOG && exit 1 || true
-echo "$0: completed"
+exit_status