LU-12567 ptlrpc: handle reply and resend reorder

[fs/lustre-release.git] / lustre / tests / sanity.sh

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 65c457e..5c88214 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -5682,38 +5682,51 @@ run_test 51b "exceed 64k subdirectory nlink limit on create, verify unlink"
 test_51d() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
        [[ $OSTCOUNT -lt 3 ]] && skip_env "needs >= 3 OSTs"
+       local qos_old
 
        test_mkdir $DIR/$tdir
+       $LFS setstripe -c $OSTCOUNT $DIR/$tdir
+
+       qos_old=$(do_facet mds1 \
+               "$LCTL get_param -n lod.$FSNAME-*.qos_threshold_rr" | head -n 1)
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "$LCTL set_param lod.$FSNAME-*.qos_threshold_rr=100"
+       stack_trap "do_nodes $(comma_list $(mdts_nodes)) \
+               '$LCTL set_param lod.$FSNAME-*.qos_threshold_rr=${qos_old%%%}'"
+
        createmany -o $DIR/$tdir/t- 1000
        $LFS getstripe $DIR/$tdir > $TMP/$tfile
-       for N in $(seq 0 $((OSTCOUNT - 1))); do
-               OBJS[$N]=$(awk -vobjs=0 '($1 == '$N') { objs += 1 } \
-                       END { printf("%0.0f", objs) }' $TMP/$tfile)
-               OBJS0[$N]=$(grep -A 1 idx $TMP/$tfile | awk -vobjs=0 \
-                       '($1 == '$N') { objs += 1 } \
-                       END { printf("%0.0f", objs) }')
-               log "OST$N has ${OBJS[$N]} objects, ${OBJS0[$N]} are index 0"
+       for ((n = 0; n < $OSTCOUNT; n++)); do
+               objs[$n]=$(awk -vobjs=0 '($1 == '$n') { objs += 1 } \
+                          END { printf("%0.0f", objs) }' $TMP/$tfile)
+               objs0[$n]=$(grep -A 1 idx $TMP/$tfile | awk -vobjs=0 \
+                           '($1 == '$n') { objs += 1 } \
+                           END { printf("%0.0f", objs) }')
+               log "OST$n has ${objs[$n]} objects, ${objs0[$n]} are index 0"
        done
        unlinkmany $DIR/$tdir/t- 1000
 
-       NLAST=0
-       for N in $(seq 1 $((OSTCOUNT - 1))); do
-               [[ ${OBJS[$N]} -lt $((${OBJS[$NLAST]} - 20)) ]] &&
-                       error "OST $N has less objects vs OST $NLAST" \
-                             " (${OBJS[$N]} < ${OBJS[$NLAST]}"
-               [[ ${OBJS[$N]} -gt $((${OBJS[$NLAST]} + 20)) ]] &&
-                       error "OST $N has less objects vs OST $NLAST" \
-                             " (${OBJS[$N]} < ${OBJS[$NLAST]}"
-
-               [[ ${OBJS0[$N]} -lt $((${OBJS0[$NLAST]} - 20)) ]] &&
-                       error "OST $N has less #0 objects vs OST $NLAST" \
-                             " (${OBJS0[$N]} < ${OBJS0[$NLAST]}"
-               [[ ${OBJS0[$N]} -gt $((${OBJS0[$NLAST]} + 20)) ]] &&
-                       error "OST $N has less #0 objects vs OST $NLAST" \
-                             " (${OBJS0[$N]} < ${OBJS0[$NLAST]}"
-               NLAST=$N
+       nlast=0
+       for ((n = 0; n < $OSTCOUNT; n++)); do
+               (( ${objs[$n]} > ${objs[$nlast]} * 4 / 5 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer objects vs. OST $nlast" \
+                             " (${objs[$n]} < ${objs[$nlast]}"; }
+               (( ${objs[$n]} < ${objs[$nlast]} * 5 / 4 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer objects vs. OST $nlast" \
+                             " (${objs[$n]} < ${objs[$nlast]}"; }
+
+               (( ${objs0[$n]} > ${objs0[$nlast]} * 4 / 5 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer #0 objects vs. OST $nlast" \
+                             " (${objs0[$n]} < ${objs0[$nlast]}"; }
+               (( ${objs0[$n]} < ${objs0[$nlast]} * 5 / 4 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer #0 objects vs. OST $nlast" \
+                             " (${objs0[$n]} < ${objs0[$nlast]}"; }
+               nlast=$n
        done
-       rm -f $TMP/$tfile
 }
 run_test 51d "check object distribution"
 
@@ -18821,26 +18834,48 @@ run_test 223 "osc reenqueue if without AGL lock granted ======================="
 
 test_224a() { # LU-1039, MRP-303
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
-
        #define OBD_FAIL_PTLRPC_CLIENT_BULK_CB   0x508
        $LCTL set_param fail_loc=0x508
-       dd if=/dev/zero of=$DIR/$tfile bs=4096 count=1 conv=fsync
+       dd if=/dev/zero of=$DIR/$tfile bs=1M count=1 conv=fsync
        $LCTL set_param fail_loc=0
        df $DIR
 }
 run_test 224a "Don't panic on bulk IO failure"
 
-test_224b() { # LU-1039, MRP-303
+test_224bd_sub() { # LU-1039, MRP-303
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
+       local timeout=$1
 
-       dd if=/dev/zero of=$DIR/$tfile bs=4096 count=1
+       shift
+       dd if=/dev/urandom of=$TMP/$tfile bs=1M count=1
+
+       $LFS setstripe -c 1 -i 0 $DIR/$tfile
+
+       dd if=$TMP/$tfile of=$DIR/$tfile bs=1M count=1
        cancel_lru_locks osc
+       set_checksums 0
+       stack_trap "set_checksums $ORIG_CSUM" EXIT
+       local at_max_saved=0
+
+       # adaptive timeouts may prevent seeing the issue
+       if at_is_enabled; then
+               at_max_saved=$(at_max_get mds)
+               at_max_set 0 mds client
+               stack_trap "at_max_set $at_max_saved mds client" EXIT
+       fi
+
        #define OBD_FAIL_PTLRPC_CLIENT_BULK_CB2   0x515
-       $LCTL set_param fail_loc=0x515
-       dd of=/dev/null if=$DIR/$tfile bs=4096 count=1
-       $LCTL set_param fail_loc=0
+       do_facet ost1 $LCTL set_param fail_val=$timeout fail_loc=0x80000515
+       dd of=$TMP/$tfile.new if=$DIR/$tfile bs=1M count=1 || "$@"
+
+       do_facet ost1 $LCTL set_param fail_loc=0
+       cmp $TMP/$tfile $TMP/$tfile.new || error "file contents wrong"
        df $DIR
 }
+
+test_224b() {
+       test_224bd_sub 3 error "dd failed"
+}
 run_test 224b "Don't panic on bulk IO failure"
 
 test_224c() { # LU-6441
@@ -18881,6 +18916,11 @@ test_224c() { # LU-6441
 }
 run_test 224c "Don't hang if one of md lost during large bulk RPC"
 
+test_224d() { # LU-11169
+       test_224bd_sub $((TIMEOUT + 2)) error "dd failed"
+}
+run_test 224d "Don't corrupt data on bulk IO timeout"
+
 MDSSURVEY=${MDSSURVEY:-$(which mds-survey 2>/dev/null || true)}
 test_225a () {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"