LU-2093 lod: fall back to RR allocation when QoS fails

[fs/lustre-release.git] / lustre / tests / sanity.sh

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index ed9e5b7..27970e0 100644 (file)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -11,6 +11,11 @@ ONLY=${ONLY:-"$*"}
 ALWAYS_EXCEPT="                27u   42a  42b  42c  42d  45   51d   68b   $SANITY_EXCEPT"
 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
 
+# with LOD/OSP landing
+# bug number for skipped tests: LU2036
+ALWAYS_EXCEPT="                 76     $ALWAYS_EXCEPT"
+
+
 # Tests that fail on uml
 CPU=`awk '/model/ {print $4}' /proc/cpuinfo`
 #                                    buffer i/o errs             sock spc runas
@@ -501,6 +506,10 @@ test_17m() {
        [ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.2.0) ] &&
        [ $(lustre_version_code $SINGLEMDS) -le $(version_code 2.2.93) ] &&
                skip "MDS 2.2.0-2.2.93 do not NUL-terminate symlinks" && return
+
+       [ "$(facet_fstype $SINGLEMDS)" != "ldiskfs" ] &&
+               skip "only for ldiskfs MDT" && return 0
+
        mkdir -p $WDIR
        long_sym=$short_sym
        # create a long symlink file
@@ -1085,6 +1094,7 @@ reset_enospc() {
        [ "$OSTIDX" ] && list=$(facet_host ost$((OSTIDX + 1)))
 
        do_nodes $list lctl set_param fail_loc=0
+       sync    # initiate all OST_DESTROYs from MDS to OST
        sleep_maxage
 }
 
@@ -2536,7 +2546,23 @@ run_test 39k "write, utime, close, stat ========================"
 # this should be set to future
 TEST_39_ATIME=`date -d "1 year" +%s`
 
+is_sles11()                                            # LU-1783
+{
+       if [ -r /etc/SuSE-release ]
+       then
+               local vers=`grep VERSION /etc/SuSE-release | awk '{print $3}'`
+               local patchlev=`grep PATCHLEVEL /etc/SuSE-release \
+                       | awk '{print $3}'`
+               if [ $vers -eq 11 ] && [ $patchlev -eq 1 ]
+               then
+                       return 0
+               fi
+       fi
+       return 1
+}
+
 test_39l() {
+       is_sles11 && skip "SLES 11 SP1" && return       # LU-1783
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        local atime_diff=$(do_facet $SINGLEMDS lctl get_param -n mdd.*.atime_diff)
 
@@ -3140,45 +3166,72 @@ run_test 51a "special situations: split htree with empty entry =="
 
 export NUMTEST=70000
 test_51b() {
-       NUMFREE=`df -i -P $DIR | tail -n 1 | awk '{ print $4 }'`
-       [ $NUMFREE -lt 21000 ] && \
-               skip "not enough free inodes ($NUMFREE)" && \
+       local BASE=$DIR/$tdir
+       mkdir -p $BASE
+
+       local mdtidx=$(printf "%04x" $($LFS getstripe -M $BASE))
+       local numfree=$(lctl get_param -n mdc.$FSNAME-MDT$mdtidx*.filesfree)
+       [ $numfree -lt 21000 ] && skip "not enough free inodes ($numfree)" &&
                return
 
-       [ $NUMFREE -lt $NUMTEST ] && NUMTEST=$(($NUMFREE - 50))
+       [ $numfree -lt $NUMTEST ] && NUMTEST=$(($numfree - 50)) &&
+               echo "reduced count to $NUMTEST due to inodes"
+
+       # need to check free space for the directories as well
+       local blkfree=$(lctl get_param -n mdc.$FSNAME-MDT$mdtidx*.kbytesavail)
+       numfree=$((blkfree / 4))
+       [ $numfree -lt $NUMTEST ] && NUMTEST=$(($numfree - 50)) &&
+               echo "reduced count to $NUMTEST due to blocks"
 
-       mkdir -p $DIR/d51b
-       createmany -d $DIR/d51b/t- $NUMTEST
+       createmany -d $BASE/d $NUMTEST && echo $NUMTEST > $BASE/fnum ||
+               echo "failed" > $BASE/fnum
 }
-run_test 51b "mkdir .../t-0 --- .../t-$NUMTEST ===================="
+run_test 51b "exceed 64k subdirectory nlink limit"
 
 test_51ba() { # LU-993
-       local BASE=$DIR/d51b
+       local BASE=$DIR/$tdir
        # unlink all but 100 subdirectories, then check it still works
        local LEFT=100
+       [ -f $BASE/fnum ] && local NUMPREV=$(cat $BASE/fnum) && rm $BASE/fnum
+
+       [ "$NUMPREV" != "failed" ] && NUMTEST=$NUMPREV
        local DELETE=$((NUMTEST - LEFT))
 
        # continue on to run this test even if 51b didn't finish,
        # just to delete the many subdirectories created.
-       ! [ -d "${BASE}/t-1" ] && skip "test_51b() not run" && return 0
+       [ ! -d "${BASE}/d1" ] && skip "test_51b() not run" && return 0
 
        # for ldiskfs the nlink count should be 1, but this is OSD specific
        # and so this is listed for informational purposes only
-       log "nlink before: $(stat -c %h $BASE)"
-       unlinkmany -d $BASE/t- $DELETE ||
-               error "unlink of first $DELETE subdirs failed"
+       echo "nlink before: $(stat -c %h $BASE), created before: $NUMTEST"
+       unlinkmany -d $BASE/d $DELETE
+       RC=$?
+
+       if [ $RC -ne 0 ]; then
+               if [ "$NUMPREV" == "failed" ]; then
+                       skip "previous setup failed"
+                       return 0
+               else
+                       error "unlink of first $DELETE subdirs failed"
+                       return $RC
+               fi
+       fi
 
-       log "nlink between: $(stat -c %h $BASE)"
-       local FOUND=$(ls -l ${BASE} | wc -l)
-       FOUND=$((FOUND - 1))  # trim the first line of ls output
+       echo "nlink between: $(stat -c %h $BASE)"
+       # trim the first line of ls output
+       local FOUND=$(($(ls -l ${BASE} | wc -l) - 1))
        [ $FOUND -ne $LEFT ] &&
                error "can't find subdirs: found only $FOUND/$LEFT"
 
-       unlinkmany -d $BASE/t- $DELETE $LEFT ||
+       unlinkmany -d $BASE/d $DELETE $LEFT ||
                error "unlink of second $LEFT subdirs failed"
-       log "nlink after: $(stat -c %h $BASE)"
+       # regardless of whether the backing filesystem tracks nlink accurately
+       # or not, the nlink count shouldn't be more than "." and ".." here
+       local AFTER=$(stat -c %h $BASE)
+       [ $AFTER -gt 2 ] && error "nlink after: $AFTER > 2" ||
+               echo "nlink after: $AFTER"
 }
-run_test 51ba "rmdir .../t-0 --- .../t-$NUMTEST"
+run_test 51ba "verify nlink for many subdirectory cleanup"
 
 test_51bb() {
        [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
@@ -3959,6 +4012,11 @@ run_test 56w "check lfs_migrate -c stripe_count works"
 
 test_57a() {
        # note test will not do anything if MDS is not local
+       if [ "$(facet_type_fstype MDS)" != ldiskfs ]; then
+               skip "Only applicable to ldiskfs-based MDTs"
+               return
+       fi
+
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        local MNTDEV="osd*.*MDT*.mntdev"
        DEV=$(do_facet $SINGLEMDS lctl get_param -n $MNTDEV)
@@ -3974,6 +4032,11 @@ test_57a() {
 run_test 57a "verify MDS filesystem created with large inodes =="
 
 test_57b() {
+       if [ "$(facet_type_fstype MDS)" != ldiskfs ]; then
+               skip "Only applicable to ldiskfs-based MDTs"
+               return
+       fi
+
        remote_mds_nodsh && skip "remote MDS with nodsh" && return
        local dir=$DIR/d57b
 
@@ -4879,6 +4942,16 @@ test_79() { # bug 12743
 run_test 79 "df report consistency check ======================="
 
 test_80() { # bug 10718
+        # relax strong synchronous semantics for slow backends like ZFS
+        local soc="obdfilter.*.sync_on_lock_cancel"
+        local soc_old=$(do_facet ost1 lctl get_param -n $soc | head -n1)
+        local hosts=
+        if [ "$soc_old" != "never" -a "$FSTYPE" != "ldiskfs" ]; then
+                hosts=$(for host in $(seq -f "ost%g" 1 $OSTCOUNT); do
+                          facet_active_host $host; done | sort -u)
+                do_nodes $hosts lctl set_param $soc=never
+        fi
+
         dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 seek=1M
         sync; sleep 1; sync
         local BEFORE=`date +%s`
@@ -4888,6 +4961,9 @@ test_80() { # bug 10718
         if [ $DIFF -gt 1 ] ; then
                 error "elapsed for 1M@1T = $DIFF"
         fi
+
+        [ -n "$hosts" ] && do_nodes $hosts lctl set_param $soc=$soc_old
+
         true
         rm -f $DIR/$tfile
 }
@@ -4897,7 +4973,7 @@ test_81a() { # LU-456
         remote_ost_nodsh && skip "remote OST with nodsh" && return
         # define OBD_FAIL_OST_MAPBLK_ENOSPC    0x228
         # MUST OR with the OBD_FAIL_ONCE (0x80000000)
-        do_facet ost0 lctl set_param fail_loc=0x80000228
+        do_facet ost1 lctl set_param fail_loc=0x80000228
 
         # write should trigger a retry and success
         $SETSTRIPE -i 0 -c 1 $DIR/$tfile
@@ -4913,7 +4989,7 @@ test_81b() { # LU-456
         remote_ost_nodsh && skip "remote OST with nodsh" && return
         # define OBD_FAIL_OST_MAPBLK_ENOSPC    0x228
         # Don't OR with the OBD_FAIL_ONCE (0x80000000)
-        do_facet ost0 lctl set_param fail_loc=0x228
+        do_facet ost1 lctl set_param fail_loc=0x228
 
         # write should retry several times and return -ENOSPC finally
         $SETSTRIPE -i 0 -c 1 $DIR/$tfile
@@ -5246,8 +5322,9 @@ test_101d() {
     echo read-ahead disabled time read $time_ra_OFF
     echo read-ahead enabled  time read $time_ra_ON
 
-    set_read_ahead $old_READAHEAD
-    rm -f $file
+       set_read_ahead $old_READAHEAD
+       rm -f $file
+       wait_delete_completed
 
     [ $time_ra_ON -lt $time_ra_OFF ] ||
         error "read-ahead enabled  time read (${time_ra_ON}s) is more than
@@ -5886,7 +5963,7 @@ free_min_max () {
        echo Max free space: OST $MAXI: $MAXV
 }
 
-test_116() {
+test_116a() { # was previously test_116()
        [ "$OSTCOUNT" -lt "2" ] && skip_env "$OSTCOUNT < 2 OSTs" && return
 
        echo -n "Free space priority "
@@ -5969,7 +6046,21 @@ test_116() {
 
        rm -rf $DIR/$tdir
 }
-run_test 116 "stripe QOS: free space balance ==================="
+run_test 116a "stripe QOS: free space balance ==================="
+
+test_116b() { # LU-2093
+#define OBD_FAIL_MDS_OSC_CREATE_FAIL     0x147
+       local old_rr
+       old_rr=$(do_facet $SINGLEMDS lctl get_param -n lov.*mdtlov*.qos_threshold_rr)
+       do_facet $SINGLEMDS lctl set_param lov.*mdtlov*.qos_threshold_rr 0
+       mkdir -p $DIR/$tdir
+       do_facet $SINGLEMDS lctl set_param fail_loc=0x147
+       createmany -o $DIR/$tdir/f- 20 || error "can't create"
+       do_facet $SINGLEMDS lctl set_param fail_loc=0
+       rm -rf $DIR/$tdir
+       do_facet $SINGLEMDS lctl set_param lov.*mdtlov*.qos_threshold_rr $old_rr
+}
+run_test 116b "QoS shouldn't LBUG if not enough OSTs found on the 2nd pass"
 
 test_117() # bug 10891
 {
@@ -6244,6 +6335,8 @@ test_118h() {
 }
 run_test 118h "Verify timeout in handling recoverables errors  =========="
 
+[ "$SLOW" = "no" ] && [ -n "$OLD_RESENDCOUNT" ] && set_resend_count $OLD_RESENDCOUNT
+
 test_118i() {
        remote_ost_nodsh && skip "remote OST with nodsh" && return
 
@@ -6282,6 +6375,8 @@ test_118i() {
 }
 run_test 118i "Fix error before timeout in recoverable error  =========="
 
+[ "$SLOW" = "no" ] && set_resend_count 4
+
 test_118j() {
        remote_ost_nodsh && skip "remote OST with nodsh" && return
 
@@ -7530,6 +7625,7 @@ test_133c() {
        $SETSTRIPE -c 1 -i 0 ${testdir}/${tfile}
        sync
        cancel_lru_locks osc
+       wait_delete_completed
 
        # clear stats.
        do_facet $SINGLEMDS $LCTL set_param mdt.*.md_stats=clear
@@ -7547,6 +7643,7 @@ test_133c() {
        check_stats ost "punch" 1
 
        rm -f ${testdir}/${tfile} || error "file remove failed"
+       wait_delete_completed
        check_stats ost "destroy" 1
 
        rm -rf $DIR/${tdir}
@@ -9067,7 +9164,7 @@ verify_jobstats() {
 
        # clear old jobstats
        do_facet $SINGLEMDS lctl set_param mdt.*.job_stats="clear"
-       do_facet ost0 lctl set_param obdfilter.*.job_stats="clear"
+       do_facet ost1 lctl set_param obdfilter.*.job_stats="clear"
 
        # use a new JobID for this test, or we might see an old one
        [ "$JOBENV" = "FAKE_JOBID" ] && FAKE_JOBID=test_id.$testnum.$RANDOM
@@ -9088,7 +9185,7 @@ verify_jobstats() {
                        grep $JOBVAL || error "No job stats found on MDT $FACET"
        fi
        if [ "$target" = "ost" -o "$target" = "both" ]; then
-               FACET=ost0
+               FACET=ost1
                do_facet $FACET lctl get_param obdfilter.*.job_stats |
                        grep $JOBVAL || error "No job stats found on OST $FACET"
        fi