X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fost-pools.sh;h=6b48a2a152576de3b660e34643b08bb08645ed07;hb=d571fd9b51d6819d58e89337b0a45ea0edb4f778;hp=81c45c563c3b3f9f2028f37ed6e5c93ef0899831;hpb=e4cf956f93a4384d19ea73e601a6651710703492;p=fs%2Flustre-release.git

diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh
index 81c45c5..6b48a2a 100644
--- a/lustre/tests/ost-pools.sh
+++ b/lustre/tests/ost-pools.sh
@@ -224,16 +224,19 @@ ost_pools_init() {
     cleanup_tests
 }
 
+set_cleanup_trap() {
+    trap "cleanup_pools $FSNAME" EXIT
+}
 
 # Initialization
 remote_mds_nodsh && skip "remote MDS with nodsh" && exit 0
 remote_ost_nodsh && skip "remote OST with nodsh" && exit 0
 ost_pools_init
 
-trap "cleanup_pools $FSNAME" EXIT
 
 # Tests for new commands added
 test_1() {
+    set_cleanup_trap
     echo "Creating a pool with a 1 character pool name"
     create_pool_nofail p
 
@@ -283,6 +286,7 @@ test_1() {
 run_test 1 "Test lctl pool_new  ========================================="
 
 test_2a() {
+    set_cleanup_trap
     destroy_pool $POOL
 
     do_facet $SINGLEMDS lctl pool_add $FSNAME.$POOL $FSNAME-OST0000 2>/dev/null
@@ -293,6 +297,7 @@ test_2a() {
 run_test 2a "pool_add: non-existant pool"
 
 test_2b() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_add $FSNAME.p1234567891234567890 \
         $FSNAME-OST0000 2>/dev/null
     [[ $? -ne 0 ]] || \
@@ -302,6 +307,7 @@ run_test 2b "pool_add: Invalid pool name"
 
 # Testing various combinations of OST name list
 test_2c() {
+    set_cleanup_trap
     local TGT
     local RC
 
@@ -356,6 +362,7 @@ test_2c() {
 run_test 2c "pool_add: OST index combinations ==========================="
 
 test_2d() {
+    set_cleanup_trap
     local TGT
     local RC
 
@@ -375,6 +382,7 @@ test_2d() {
 run_test 2d "pool_add: OSTs that don't exist should be rejected ========"
 
 test_2e() {
+    set_cleanup_trap
     local TGT
     local RC
     local RESULT
@@ -405,6 +413,7 @@ test_2e() {
 run_test 2e "pool_add: OST already in a pool should be rejected ========"
 
 test_3a() {
+    set_cleanup_trap
     lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         destroy_pool $POOL
@@ -416,6 +425,7 @@ test_3a() {
 run_test 3a "pool_remove: non-existant pool"
 
 test_3b() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_remove ${NON_EXISTANT_FS}.$POOL OST0000 2>/dev/null
     [[ $? -ne 0 ]] || \
         error "pool_remove did not fail even though fsname did not exist."
@@ -423,6 +433,7 @@ test_3b() {
 run_test 3b "pool_remove: non-existant fsname"
 
 test_3c() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_remove $FSNAME.p1234567891234567890 \
         $FSNAME-OST0000 2>/dev/null
     [[ $? -ne 0 ]] || \
@@ -432,6 +443,7 @@ run_test 3c "pool_remove: Invalid pool name"
 
 # Testing various combinations of OST name list
 test_3d() {
+    set_cleanup_trap
     lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         destroy_pool $POOL
@@ -466,6 +478,7 @@ test_3d() {
 run_test 3d "pool_remove: OST index combinations ==========================="
 
 test_4a() {
+    set_cleanup_trap
     lctl get_param -n lov.$FSNAME-*.pools.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         destroy_pool $POOL
@@ -477,6 +490,7 @@ test_4a() {
 run_test 4a "pool_destroy: non-existant pool"
 
 test_4b() {
+    set_cleanup_trap
     do_facet $SINGLEMDS lctl pool_destroy ${NON_EXISTANT_FS}.$POOL 2>/dev/null
     [[ $? -ne 0 ]] || \
         error "pool_destroy did not fail even though the filesystem did not exist."
@@ -484,6 +498,7 @@ test_4b() {
 run_test 4b "pool_destroy: non-existant fs-name"
 
 test_4c() {
+    set_cleanup_trap
     create_pool_nofail $POOL
     add_pool $POOL "OST0000" "$FSNAME-OST0000_UUID "
 
@@ -546,6 +561,7 @@ sub_test_5() {
 }
 
 test_5() {
+    set_cleanup_trap
     # Issue commands from client
     sub_test_5 $LCTL
     sub_test_5 $LFS
@@ -558,6 +574,7 @@ test_5() {
 run_test 5 "lfs/lctl pool_list"
 
 test_6() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local POOL_DIR=$POOL_ROOT/dir_tst
     local POOL_FILE=$POOL_ROOT/file_tst
@@ -602,12 +619,11 @@ test_6() {
     [[ $? -ne 0 ]] || \
         error "$SETSTRIPE with start index outside the pool did not fail."
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
 }
 run_test 6 "getstripe/setstripe"
 
 test_11() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
 
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
@@ -642,14 +658,12 @@ test_11() {
 
     rm -rf $POOL_ROOT/dir?
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
-
     return 0
 }
 run_test 11 "OSTs in overlapping/multiple pools"
 
 test_12() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
 
     [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
@@ -699,14 +713,12 @@ test_12() {
     check_file_in_pool $POOL_ROOT/file3 $POOL
     check_file_in_pool $POOL_ROOT/file4 $POOL2
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
-
     return 0    
 }
 run_test 12 "OST Pool Membership"
 
 test_13() {
+    set_cleanup_trap
     [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
 
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
@@ -763,13 +775,13 @@ test_13() {
     done
 
     rm -rf create_dir $POOL_ROOT/dir?
-    destroy_pool $POOL
 
     return 0
 }
 run_test 13 "Striping characteristics in a pool"
 
 test_14() {
+    set_cleanup_trap
     [[ $OSTCOUNT -le 2 ]] && skip_env "Need atleast 3 OSTs" && return
 
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
@@ -797,24 +809,24 @@ test_14() {
 
         # echo "Iteration: $i OST: $OST"
         create_file $POOL_ROOT/dir1/file${i} $POOL 1
-        check_file_in_osts $POOL_ROOT/dir1/file${i} $OST
+        check_file_in_pool $POOL_ROOT/dir1/file${i} $POOL
         i=$((i+1))
     done
 
-    # Fill OST $TGT_FIRST with 10M files
+    # Fill up OST0 until it is nearly full.
+    # Create 9 files of size OST0_SIZE/10 each.
     create_dir $POOL_ROOT/dir2 $POOL2 1
-    RC=0
-    i=0
-    while [[ $RC -eq 0 ]];
+    $LFS df $POOL_ROOT/dir2
+    echo "Filling up OST0"
+    OST0_SIZE=`$LFS df $POOL_ROOT/dir2 | awk '/\[OST:0\]/ {print $4}'`
+    FILE_SIZE=$((OST0_SIZE/1024/10))
+    i=1
+    while [[ $i -lt 10 ]];
     do
-      dd if=/dev/zero of=$POOL_ROOT/dir2/f${i} bs=1k count=$((1024*10))
-      RC=$?
+      dd if=/dev/zero of=$POOL_ROOT/dir2/f${i} bs=1M count=$FILE_SIZE
       i=$((i+1))
     done
-
-    # Leave some space on the OST
-    rm -f $POOL_ROOT/dir2/f0
-    df -h /mnt/ost?
+    $LFS df $POOL_ROOT/dir2
 
     # OST $TGT_FIRST is no longer favored; but it may still be used.
     create_dir $POOL_ROOT/dir3 $POOL 1
@@ -826,13 +838,12 @@ test_14() {
     done
 
     rm -rf $POOL_ROOT
-    destroy_pool $POOL
-    destroy_pool $POOL2
     return 0
 }
 run_test 14 "Round robin and QOS striping within a pool"
 
 test_15() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=100
     local i=0
@@ -854,18 +865,12 @@ test_15() {
       i=$((i+1))
     done
 
-    i=0
-    while [[ $i -lt $OSTCOUNT ]]
-    do
-      destroy_pool pool${i}
-      i=$((i+1))
-    done
-
     return 0
 }
 run_test 15 "One directory per OST/pool"
 
 test_16() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=10
     local i=0
@@ -891,13 +896,12 @@ test_16() {
 
     rm -rf $POOL_ROOT/$tdir
 
-    destroy_pool $POOL
-
     return 0
 }
 run_test 16 "Inheritance of pool properties"
 
 test_17() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=10
     local i=0
@@ -926,72 +930,83 @@ test_17() {
 }
 run_test 17 "Referencing an empty pool"
 
-create_perf_one() {
-    local dir=$1/d
+create_perf() {
+    local cdir=$1/d
     local numfiles=$2
     local time
 
-    mkdir -p $dir
+    mkdir -p $cdir
     sync; sleep 5 # give pending IO a chance to go to disk
-    stat=$(createmany -o $dir/${tfile} $numfiles)
-    rm -rf $dir
+    stat=$(createmany -o $cdir/${tfile} $numfiles)
+    rm -rf $cdir
     sync
     time=$(echo $stat | cut -f 5 -d ' ')
-    echo $stat > /dev/stderr
+    echo $stat >> /dev/stderr
     echo $time
 }
 
-create_perf() {
-    local t1=$(create_perf_one $1 $2)
-    local t2=$(create_perf_one $1 $2)
-    local t3=$(create_perf_one $1 $2)
-    local time=$(echo "scale=2; ( $t1 + $t2 + $t3 ) / 3" | bc)
-
-    echo "$time"
-}
-
 test_18() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
-    local numfiles=10000
-    local i=0
-    local time1
-    local time2
-    local time3
-    local dir=$POOL_ROOT/dir
-    local stat
-    local diff
-
-    time1=$(create_perf $dir $numfiles)
-
-    create_pool_nofail $POOL
-
-    add_pool $POOL $TGT_ALL "$TGT_UUID"
-
-    create_dir $dir $POOL
-    time2=$(create_perf $dir $numfiles)
-
-    destroy_pool $POOL
-
-    time3=$(create_perf $dir $numfiles)
-
-    echo Time taken for $numfiles creates without pools: $time1
-    echo Time taken for $numfiles creates with pools: $time2
-    echo Time taken for $numfiles creates without pools: $time3
-
-    deg=$(echo "scale=2; (($time2 - $time3) * 100 / $time3) > 5" | bc)
-    diff=$(echo "scale=2; ($time2 - $time3) * 100 / $time3" | bc)
+    local numfiles=9877
+    local plaindir=$POOL_ROOT/plaindir
+    local pooldir=$POOL_ROOT/pooldir
+	local t1=0
+	local t2=0
+	local t3=0
+	local diff
+
+    for i in $(seq 1 3);
+    do
+        echo "Create performance, iteration $i, $numfiles files x 3"
+
+		time1=$(create_perf $plaindir $numfiles)
+		echo "iter $i: $numfiles creates without pool: $time1"
+		t1=$(echo "scale=2; $t1 + $time1" | bc)
+
+		create_pool_nofail $POOL > /dev/null
+		add_pool $POOL $TGT_ALL "$TGT_UUID" > /dev/null
+		create_dir $pooldir $POOL
+		time2=$(create_perf $pooldir $numfiles)
+		echo "iter $i: $numfiles creates with pool: $time2"
+		t2=$(echo "scale=2; $t2 + $time2" | bc)
+
+		destroy_pool $POOL > /dev/null
+		time3=$(create_perf $pooldir $numfiles)
+		echo "iter $i: $numfiles creates with missing pool: $time3"
+		t3=$(echo "scale=2; $t3 + $time3" | bc)
+
+		echo
+	done
+
+	time1=$(echo "scale=2; $t1 / $i" | bc)
+	echo Avg time taken for $numfiles creates without pool: $time1
+	time2=$(echo "scale=2; $t2 / $i" | bc)
+	echo Avg time taken for $numfiles creates with pool: $time2
+	time3=$(echo "scale=2; $t3 / $i" | bc)
+	echo Avg time taken for $numfiles creates with missing pool: $time3
+
+	# Set this high until we establish a baseline for what the degradation
+	# is / should be
+	max=30
+    diff=$(echo "scale=2; ($time2 - $time1) * 100 / $time1" | bc)
+    echo  "No pool to wide pool: $diff %."
+    deg=$(echo "scale=2; $diff > $max" | bc)
+    [ "$deg" == "1" ] && error_ignore 23408  "Degradation with wide pool is $diff % (> $max %)"
+
+	max=15
+    diff=$(echo "scale=2; ($time3 - $time1) * 100 / $time1" | bc)
+    echo  "No pool to missing pool: $diff %."
+    deg=$(echo "scale=2; $diff > $max" | bc)
+    [ "$deg" == "1" ] && error_ignore 23408 "Degradation with missing pool is $diff % (> $max %)"
 
-    if [[ "$deg" == "1" ]]; then
-        error "Performance degradation with pools is $diff %."
-    else
-        echo  "Performance degradation with pools is $diff %."
-    fi
     return 0
 }
 run_test 18 "File create in a directory which references a deleted pool"
 
 
 test_19() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=12
     local dir1=$POOL_ROOT/dir1
@@ -1017,13 +1032,13 @@ test_19() {
     done
 
     rm -rf $dir1 $dir2
-    destroy_pool $POOL
 
     return 0
 }
 run_test 19 "Pools should not come into play when not specified"
 
 test_20() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     local numfiles=12
     local dir1=$POOL_ROOT/dir1
@@ -1064,14 +1079,13 @@ test_20() {
     check_file_not_in_pool $dir2/file4 $POOL2
 
     rm -rf $dir1
-    destroy_pool $POOL
-    destroy_pool $POOL2
 
     return 0
 }
 run_test 20 "Different pools in a directory hierarchy."
 
 test_21() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
 
@@ -1089,7 +1103,6 @@ test_21() {
     check_file_in_pool $dir/file1 $POOL
 
     rm -rf $dir
-    destroy_pool $POOL
 
     return 0
 }
@@ -1114,6 +1127,7 @@ add_loop() {
 }
 
 test_22() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need at least 2 OSTs" && return
 
@@ -1132,14 +1146,12 @@ test_22() {
           error "createmany $POOL_ROOT/${tfile} failed!"
     wait
 
-    destroy_pool $POOL
-    destroy_pool $POOL2
-
     return 0
 }
 run_test 22 "Simultaneous manipulation of a pool"
 
 test_23() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
 
@@ -1215,13 +1227,13 @@ test_23() {
     df -h
 
     rm -rf $POOL_ROOT
-    destroy_pool $POOL
 
     return 0
 }
 run_test 23 "OST pools and quota"
 
 test_24() {
+    set_cleanup_trap
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
     [[ $OSTCOUNT -le 1 ]] && skip_env "Need atleast 2 OSTs" && return
 
@@ -1295,13 +1307,13 @@ test_24() {
     done
 
     rm -rf $POOL_ROOT
-    destroy_pool $POOL
 
     return 0
 }
 run_test 24 "Independence of pool from other setstripe parameters"
 
 test_25() {
+    set_cleanup_trap
     local dev=$(mdsdevname ${SINGLEMDS//mds/})
     local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
 
@@ -1309,25 +1321,76 @@ test_25() {
 
     for i in $(seq 10); do
         create_pool_nofail pool$i
-	do_facet $SINGLEMDS lctl pool_add $FSNAME.pool$i OST0000
+        do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool$i OST0000; sync"
+        wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool$i | \
+            sort -u | tr '\n' ' ' " "$FSNAME-OST0000_UUID " || \
+            error "pool_add failed: $1; $2"
+
         stop $SINGLEMDS || return 1
         start $SINGLEMDS ${dev} $MDS_MOUNT_OPTS  || \
             { error "Failed to start $SINGLEMDS after stopping" && break; }
-	clients_up
+        wait_osc_import_state mds ost FULL
+        clients_up
 
+        wait_mds_ost_sync 
         # Veriy that the pool got created and is usable
-        echo "Creating a file in pool$i"
-        create_file $POOL_ROOT/file$i pool$i || break
-        check_file_in_pool $POOL_ROOT/file$i pool$i || break
+        df $POOL_ROOT > /dev/null
+        sleep 5
+        # Make sure OST0 can be striped on
+        $SETSTRIPE -o 0 -c 1 $POOL_ROOT/$tfile
+        STR=$($GETSTRIPE $POOL_ROOT/$tfile | grep 0x | cut -f2 | tr -d " ")
+        rm $POOL_ROOT/$tfile
+        if [[ "$STR" == "0" ]]; then
+                echo "Creating a file in pool$i"
+                create_file $POOL_ROOT/file$i pool$i || break
+                check_file_in_pool $POOL_ROOT/file$i pool$i || break
+        else
+                echo "OST 0 seems to be unavailable.  Try later."
+        fi
     done
 
     rm -rf $POOL_ROOT
-    for i in $(seq 10); do
-        destroy_pool pool$i
-    done
 }
 run_test 25 "Create new pool and restart MDS ======================="
 
+test_26() {
+    local OSTCOUNT=`lctl get_param -n lov.$LOVNAME.*clilov*.numobd`
+    [[ $OSTCOUNT -le 2 ]] && skip_env "Need at least 3 OSTs" && return
+    set_cleanup_trap
+    local dev=$(mdsdevname ${SINGLEMDS//mds/})
+    local POOL_ROOT=${POOL_ROOT:-$DIR/$tdir}
+
+    mkdir -p $POOL_ROOT
+
+    create_pool_nofail $POOL 
+ 
+    do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0000; sync"
+    wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
+    sort -u | grep $FSNAME-OST0000_UUID " "$FSNAME-OST0000_UUID" || \
+    error "pool_add failed: $1; $2"
+  
+    do_facet $SINGLEMDS "lctl pool_add $FSNAME.pool1 OST0002; sync"
+    wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.pool1 | \
+    sort -u | grep $FSNAME-OST0002_UUID" "$FSNAME-OST0002_UUID" || \
+    error "pool_add failed: $1; $2"
+  
+    # Veriy that the pool got created and is usable
+    df $POOL_ROOT
+    echo "Creating files in $POOL"
+
+    for ((i=0;i<10;i++)); do
+        #OBD_FAIL_MDS_OSC_CREATE_FAIL     0x147  
+        #Fail OST0000 to make sure the objects create on the other ost in the pool 
+        do_facet $SINGLEMDS lctl set_param fail_loc=0x147
+        do_facet $SINGLEMDS lctl set_param fail_val=0
+        create_file $POOL_ROOT/file$i $POOL 1 -1 || break
+        do_facet $SINGLEMDS lctl set_param fail_loc=0
+        check_file_in_pool $POOL_ROOT/file$i $POOL || break
+    done
+    rm -rf $POOL_ROOT
+}
+run_test 26 "Choose other OSTs in the pool first in the creation remedy"
+
 log "cleanup: ======================================================"
 cd $ORIG_PWD
 cleanup_pools $FSNAME