LU-17990 tests: sanity 33hh MDT index match often

test_33hh in sanity.sh failed likely due to random chance as
occasionally the generation of names will only contain only numbers
or only letters.

To reduce the chance of this being an issue, if the test fails it
will re-run up to 3 times internally, after which if there is still
an issue something is surely wrong and it will fail.

Test-Parameters: trivial testlist=sanity env=ONLY=33hh,ONLY_REPEAT=100
Test-Parameters: trivial testlist=sanity env=ONLY=33hh,ONLY_REPEAT=100

Signed-off-by: Frederick Dilger <fdilger@whamcloud.com>
Change-Id: I4385bd2621f1305e9c11b27f9eb67f9a45aa606a
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55611
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Lai Siyao <lai.siyao@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>

diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 867d2e7..ff59e9d 100755 (executable)
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -4478,6 +4478,7 @@ sub_33h() {
        local failed=0
        local patterns=(".$tfile.XXXXXX" "$tfile.XXXXXXXX")
        local pattern
+       local stat
 
        for pattern in ${patterns[*]}; do
                echo "pattern $pattern"
@@ -4504,55 +4505,71 @@ sub_33h() {
        [[ "$hash_type" == "crush" ]] && expect=$count ||
                expect=$((count / MDSCOUNT))
 
-       # crush2 doesn't put all-numeric suffixes on the same MDT,
-       # filename like $tfile.12345678 should *not* be considered temp
-       for pattern in ${patterns[*]}; do
-               local base=${pattern%%X*}
-               local suff=${pattern#$base}
+       local tries=3
 
-               echo "pattern $pattern"
-               for (( i = 0; i < $count; i++ )); do
-                       fname=$DIR/$tdir/$base$((${suff//X/1} + i))
-                       touch $fname || error "touch $fname failed"
-                       index2=$($LFS getstripe -m $fname)
-                       (( $index != $index2 )) && continue
+       for (( try = tries - 1; try >= 0; try-- )); do
+               # crush2 doesn't put all-numeric suffixes on the same MDT,
+               # filename like $tfile.12345678 should *not* be considered temp
+               for pattern in ${patterns[*]}; do
+                       local base=${pattern%%X*}
+                       local suff=${pattern#$base}
 
-                       same=$((same + 1))
-               done
-       done
+                       echo "pattern $pattern"
+                       for (( i = 0; i < $count; i++ )); do
+                               fname=$DIR/$tdir/$base$((${suff//X/1} + i))
+                               touch $fname || error "touch $fname failed"
+                               index2=$($LFS getstripe -m $fname)
+                               (( $index != $index2 )) && continue
 
-       # the number of "bad" hashes is random, as it depends on the random
-       # filenames generated by "mktemp".  Allow some margin in the results.
-       echo "$((same/${#patterns[*]}))/$count matches, expect ~$expect for $1"
-       (( same / ${#patterns[*]} <= expect * 9 / 7 &&
-          same / ${#patterns[*]} > expect * 5 / 7 )) ||
-               error "MDT index match $((same / ${#patterns[*]}))/$count times"
-       same=0
+                               same=$((same + 1))
+                       done
+               done
 
-       # crush2 doesn't put suffixes with special characters on the same MDT
-       # filename like $tfile.txt.1234 should *not* be considered temp
-       for pattern in ${patterns[*]}; do
-               local base=${pattern%%X*}
-               local suff=${pattern#$base}
+               stat="$((same/${#patterns[*]}))/$count"
+               # the number of "bad" hashes is random, as it depends on the
+               # random filenames generated by "mktemp". Allow some margin in
+               # the results.
+               echo "$stat matches, expect ~$expect for $1"
+               (( same / ${#patterns[*]} <= expect * 9 / 7 &&
+                  same / ${#patterns[*]} > expect * 5 / 7 )) && same=0 && break
+               log "MDT index match $stat times, $try tries left"
+
+               (( try > 0 )) ||
+               error "MDT index match $stat times, after $tries tries"
+               same=0
+       done
+
+       for (( try = tries - 1; try >= 0; try-- )); do
+               # crush2 doesn't put suffixes with special chars on the same MDT
+               # filenames like $tfile.txt.1234 should *not* be considered temp
+               for pattern in ${patterns[*]}; do
+                       local base=${pattern%%X*}
+                       local suff=${pattern#$base}
+
+                       pattern=$base...${suff/XXX}
+                       echo "pattern=$pattern"
+                       for (( i = 0; i < $count; i++ )); do
+                               fname=$(mktemp $DIR/$tdir/$pattern) ||
+                                       error "touch $fname failed"
+                               index2=$($LFS getstripe -m $fname)
+                               (( $index != $index2 )) && continue
+
+                               same=$((same + 1))
+                       done
+               done
 
-               pattern=$base...${suff/XXX}
-               echo "pattern=$pattern"
-               for (( i = 0; i < $count; i++ )); do
-                       fname=$(mktemp $DIR/$tdir/$pattern) ||
-                               error "touch $fname failed"
-                       index2=$($LFS getstripe -m $fname)
-                       (( $index != $index2 )) && continue
+               stat="$((same/${#patterns[*]}))/$count"
+               # the number of "bad" hashes is random, as it depends on the
+               # filenames generated by "mktemp".  Allow some margin in results
+               echo "$stat matches, expect ~$expect for $1"
+               (( same / ${#patterns[*]} <= expect * 9 / 7 &&
+                  same / ${#patterns[*]} > expect * 5 / 7 )) && break
+               log "MDT index match $stat times, $try tries left"
 
-                       same=$((same + 1))
-               done
+               (( try > 0 )) ||
+               error "MDT index match $stat times, after $tries tries"
+               same=0
        done
-
-       # the number of "bad" hashes is random, as it depends on the random
-       # filenames generated by "mktemp".  Allow some margin in the results.
-       echo "$((same/${#patterns[*]}))/$count matches, expect ~$expect for $1"
-       (( same / ${#patterns[*]} <= expect * 9 / 7 &&
-          same / ${#patterns[*]} > expect * 5 / 7 )) ||
-               error "MDT index match $((same / ${#patterns[*]}))/$count times"
 }
 
 test_33h() {