LU-12058 tests: improve sanity test_51d reliability

The original commit message (b=10671, not in git history) stated:

    When selecting which OSTs to stripe files over, for files with
    a stripe count that divides evenly into the number of OSTs,
    the MDS is always picking the same starting OST for each file.
    Return the OST selection heuristic to the original design.

This test is mainly to catch logic errors in the object allocation
code, not to achieve perfect balance across all OSTs.

Firstly, fix the test to actually verify stripe-0 precession works.
This needs stripe_count=$OSTCOUNT, which was once the test default.

Make the test more robust by disabling QOS to give a more uniform
distribution of files across OSTs, even if they are space imbalanced.

Increase the threshold of error to reduce sensitivity to allocation
imbalances due to fewer preallocated objects available on the MDS.

Lustre-change: https://review.whamcloud.com/44762
Lustre-commit: 2e832a0038af7f3d9f022beb862578988592d682

Test-Parameters: trivial
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Change-Id: I21f80ebb6f51e72bf4a5b19abe497ee9797a616a
Reviewed-by: Arshad Hussain <arshad.hussain@aeoncomputing.com>
Reviewed-by: Vikentsi Lapa <vlapa@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/46272
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 9f7cf85..76b39c2 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -5471,38 +5471,51 @@ run_test 51b "exceed 64k subdirectory nlink limit on create, verify unlink"
 test_51d() {
        [ $PARALLEL == "yes" ] && skip "skip parallel run"
        [[ $OSTCOUNT -lt 3 ]] && skip_env "needs >= 3 OSTs"
+       local qos_old
 
        test_mkdir $DIR/$tdir
+       $LFS setstripe -c $OSTCOUNT $DIR/$tdir
+
+       qos_old=$(do_facet mds1 \
+               "$LCTL get_param -n lod.$FSNAME-*.qos_threshold_rr" | head -n 1)
+       do_nodes $(comma_list $(mdts_nodes)) \
+               "$LCTL set_param lod.$FSNAME-*.qos_threshold_rr=100"
+       stack_trap "do_nodes $(comma_list $(mdts_nodes)) \
+               '$LCTL set_param lod.$FSNAME-*.qos_threshold_rr=${qos_old%%%}'"
+
        createmany -o $DIR/$tdir/t- 1000
        $LFS getstripe $DIR/$tdir > $TMP/$tfile
-       for N in $(seq 0 $((OSTCOUNT - 1))); do
-               OBJS[$N]=$(awk -vobjs=0 '($1 == '$N') { objs += 1 } \
-                       END { printf("%0.0f", objs) }' $TMP/$tfile)
-               OBJS0[$N]=$(grep -A 1 idx $TMP/$tfile | awk -vobjs=0 \
-                       '($1 == '$N') { objs += 1 } \
-                       END { printf("%0.0f", objs) }')
-               log "OST$N has ${OBJS[$N]} objects, ${OBJS0[$N]} are index 0"
+       for ((n = 0; n < $OSTCOUNT; n++)); do
+               objs[$n]=$(awk -vobjs=0 '($1 == '$n') { objs += 1 } \
+                          END { printf("%0.0f", objs) }' $TMP/$tfile)
+               objs0[$n]=$(grep -A 1 idx $TMP/$tfile | awk -vobjs=0 \
+                           '($1 == '$n') { objs += 1 } \
+                           END { printf("%0.0f", objs) }')
+               log "OST$n has ${objs[$n]} objects, ${objs0[$n]} are index 0"
        done
        unlinkmany $DIR/$tdir/t- 1000
 
-       NLAST=0
-       for N in $(seq 1 $((OSTCOUNT - 1))); do
-               [[ ${OBJS[$N]} -lt $((${OBJS[$NLAST]} - 20)) ]] &&
-                       error "OST $N has less objects vs OST $NLAST" \
-                             " (${OBJS[$N]} < ${OBJS[$NLAST]}"
-               [[ ${OBJS[$N]} -gt $((${OBJS[$NLAST]} + 20)) ]] &&
-                       error "OST $N has less objects vs OST $NLAST" \
-                             " (${OBJS[$N]} < ${OBJS[$NLAST]}"
-
-               [[ ${OBJS0[$N]} -lt $((${OBJS0[$NLAST]} - 20)) ]] &&
-                       error "OST $N has less #0 objects vs OST $NLAST" \
-                             " (${OBJS0[$N]} < ${OBJS0[$NLAST]}"
-               [[ ${OBJS0[$N]} -gt $((${OBJS0[$NLAST]} + 20)) ]] &&
-                       error "OST $N has less #0 objects vs OST $NLAST" \
-                             " (${OBJS0[$N]} < ${OBJS0[$NLAST]}"
-               NLAST=$N
+       nlast=0
+       for ((n = 0; n < $OSTCOUNT; n++)); do
+               (( ${objs[$n]} > ${objs[$nlast]} * 4 / 5 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer objects vs. OST $nlast" \
+                             " (${objs[$n]} < ${objs[$nlast]}"; }
+               (( ${objs[$n]} < ${objs[$nlast]} * 5 / 4 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer objects vs. OST $nlast" \
+                             " (${objs[$n]} < ${objs[$nlast]}"; }
+
+               (( ${objs0[$n]} > ${objs0[$nlast]} * 4 / 5 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer #0 objects vs. OST $nlast" \
+                             " (${objs0[$n]} < ${objs0[$nlast]}"; }
+               (( ${objs0[$n]} < ${objs0[$nlast]} * 5 / 4 )) ||
+                       { $LFS df && $LFS df -i &&
+                       error "OST $n has fewer #0 objects vs. OST $nlast" \
+                             " (${objs0[$n]} < ${objs0[$nlast]}"; }
+               nlast=$n
        done
-       rm -f $TMP/$tfile
 }
 run_test 51d "check object distribution"