From: James Nunez Date: Fri, 26 Jul 2013 15:41:43 +0000 (-0600) Subject: LU-3640 test: Sanity test 116a fails when an OST is full X-Git-Tag: 2.5.0~36 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=d1d5446befc4cef4b6832e9b1708f9314f6adae7;p=fs%2Flustre-release.git LU-3640 test: Sanity test 116a fails when an OST is full Sanity test 116a looks for the OST with the least amount of free space and fills it to varying levels. It does not check to see if the OST is full and fails because of a divide by zero (bytes available on full OST) error. This patch adds a check to see if an OST is full and, if so, the test is now skipped. We also corrected where to look for the qos_prio_free file from the client to the MDS. A correction was made to how the number of files written to each OST is calculated. We also added using qos_threshold_rr to check how uneven the OSTs need to be. This test is usually skipped since filling large OST takes too much time. In order to get this test to run faster, we’ve added 1. A check to see if the OST needs to be filled to create uneven OSTs. For QOS mode, a pair of OSTs must be at least qos_threshold_rr% off balanced. If the OSTs are already this uneven, we skip writing files to an OST. If the OSTs do not exceed this threshold, we write only 2% over the threshold amount to create uneven OSTs. 2. A cap on the number of files written when checking QOS functionality. The files are small, 200k in size,and we were writing enough files to fill 25% of the most full OST. Now we cap the number of file written to at most 600. There’s no meaning to 600 files, it just seemed like enough files to make sure that the OST that has the most space available is getting more files. We could make this number of files dependent on the number of OSTs. Signed-off-by: James Nunez Change-Id: Ide1565015ffc26a849adabd0d06285971bcee9f2 Reviewed-on: http://review.whamcloud.com/7132 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: Andreas Dilger Reviewed-by: Emoly Liu --- diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 9280dbd..858573a 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -6614,53 +6614,79 @@ test_116a() { # was previously test_116() [ "$OSTCOUNT" -lt "2" ] && skip_env "$OSTCOUNT < 2 OSTs" && return echo -n "Free space priority " - lctl get_param -n lov.*-clilov-*.qos_prio_free + do_facet $SINGLEMDS lctl get_param -n lo*.*-mdtlov.qos_prio_free | + head -1 declare -a AVAIL free_min_max - [ $MINV -gt 960000 ] && skip "too much free space in OST$MINI, skip" &&\ - return - # generate uneven OSTs + [ $MINV -eq 0 ] && skip "no free space in OST$MINI, skip" && return + trap simple_cleanup_common EXIT + + # Check if we need to generate uneven OSTs test_mkdir -p $DIR/$tdir/OST${MINI} - declare -i FILL - FILL=$(($MINV / 4)) - echo "Filling 25% remaining space in OST${MINI} with ${FILL}Kb" - $SETSTRIPE -i $MINI -c 1 $DIR/$tdir/OST${MINI}||error "setstripe failed" - i=0 - while [ $FILL -gt 0 ]; do - i=$(($i + 1)) - dd if=/dev/zero of=$DIR/$tdir/OST${MINI}/$tfile-$i bs=2M count=1 2>/dev/null - FILL=$(($FILL - 2048)) - echo -n . - done - FILL=$(($MINV / 4)) - sync - sleep_maxage + local FILL=$(($MINV / 4)) + local DIFF=$(($MAXV - $MINV)) + local DIFF2=$(($DIFF * 100 / $MINV)) + + local threshold=$(do_facet $SINGLEMDS \ + lctl get_param -n *.*MDT0000-mdtlov.qos_threshold_rr | head -1) + threshold=${threshold%%%} + echo -n "Check for uneven OSTs: " + echo -n "diff=${DIFF}KB (${DIFF2}%) must be > ${threshold}% ..." + + if [ $DIFF2 -gt $threshold ]; then + echo "ok" + echo "Don't need to fill OST$MINI" + else + # generate uneven OSTs. Write 2% over the QOS threshold value + echo "no" + DIFF=$(($threshold - $DIFF2 + 2)) + DIFF2=$(( ($MINV * $DIFF)/100 )) + echo "Fill ${DIFF}% remaining space in OST${MINI} with ${DIFF2}KB" + $SETSTRIPE -i $MINI -c 1 $DIR/$tdir/OST${MINI} || + error "setstripe failed" + DIFF=$(($DIFF2 / 2048)) + i=0 + while [ $i -lt $DIFF ]; do + i=$(($i + 1)) + dd if=/dev/zero of=$DIR/$tdir/OST${MINI}/$tfile-$i \ + bs=2M count=1 2>/dev/null + echo -n . + done + echo . + sync + sleep_maxage + free_min_max + fi - free_min_max DIFF=$(($MAXV - $MINV)) DIFF2=$(($DIFF * 100 / $MINV)) - echo -n "diff=${DIFF}=${DIFF2}% must be > 20% for QOS mode..." - if [ $DIFF2 -gt 20 ]; then - echo "ok" + echo -n "diff=${DIFF}=${DIFF2}% must be > ${threshold}% for QOS mode..." + if [ $DIFF2 -gt $threshold ]; then + echo "ok" else - echo "failed - QOS mode won't be used" - error_ignore "QOS imbalance criteria not met" - return + echo "failed - QOS mode won't be used" + error_ignore 0000 "QOS imbalance criteria not met" + simple_cleanup_common + return fi MINI1=$MINI; MINV1=$MINV MAXI1=$MAXI; MAXV1=$MAXV # now fill using QOS - echo writing a bunch of files to QOS-assigned OSTs $SETSTRIPE -c 1 $DIR/$tdir + FILL=$(($FILL / 200)) + if [ $FILL -gt 600 ]; then + FILL=600 + fi + echo "writing $FILL files to QOS-assigned OSTs" i=0 - while [ $FILL -gt 0 ]; do - i=$(($i + 1)) - dd if=/dev/zero of=$DIR/$tdir/$tfile-$i bs=1024 count=200 2>/dev/null - FILL=$(($FILL - 200)) - echo -n . + while [ $i -lt $FILL ]; do + i=$(($i + 1)) + dd if=/dev/zero of=$DIR/$tdir/$tfile-$i bs=200k \ + count=1 2>/dev/null + echo -n . done echo "wrote $i 200k files" sync @@ -6672,26 +6698,31 @@ test_116a() { # was previously test_116() echo "free space delta: orig $DIFF final $DIFF2" [ $DIFF2 -gt $DIFF ] && echo "delta got worse!" DIFF=$(($MINV1 - ${AVAIL[$MINI1]})) - echo "Wrote $DIFF to smaller OST $MINI1" + echo "Wrote ${DIFF}KB to smaller OST $MINI1" DIFF2=$(($MAXV1 - ${AVAIL[$MAXI1]})) - echo "Wrote $DIFF2 to larger OST $MAXI1" - [ $DIFF -gt 0 ] && echo "Wrote $(($DIFF2 * 100 / $DIFF - 100))% more data to larger OST $MAXI1" + echo "Wrote ${DIFF2}KB to larger OST $MAXI1" + FILL=$(($DIFF2 * 100 / $DIFF - 100)) + [ $DIFF -gt 0 ] && + echo "Wrote ${FILL}% more data to larger OST $MAXI1" # Figure out which files were written where UUID=$(lctl get_param -n lov.${FSNAME}-clilov-*.target_obd | - awk '/'$MINI1': / {print $2; exit}') + awk '/'$MINI1': / {print $2; exit}') echo $UUID - MINC=$($GETSTRIPE --obd $UUID $DIR/$tdir | wc -l) + MINC=$($GETSTRIPE --ost $UUID $DIR/$tdir | grep $DIR | wc -l) echo "$MINC files created on smaller OST $MINI1" UUID=$(lctl get_param -n lov.${FSNAME}-clilov-*.target_obd | - awk '/'$MAXI1': / {print $2; exit}') + awk '/'$MAXI1': / {print $2; exit}') echo $UUID - MAXC=$($GETSTRIPE --obd $UUID $DIR/$tdir | wc -l) + MAXC=$($GETSTRIPE --ost $UUID $DIR/$tdir | grep $DIR | wc -l) echo "$MAXC files created on larger OST $MAXI1" - [ $MINC -gt 0 ] && echo "Wrote $(($MAXC * 100 / $MINC - 100))% more files to larger OST $MAXI1" - [ $MAXC -gt $MINC ] || error_ignore "stripe QOS didn't balance free space" + FILL=$(($MAXC * 100 / $MINC - 100)) + [ $MINC -gt 0 ] && + echo "Wrote ${FILL}% more files to larger OST $MAXI1" + [ $MAXC -gt $MINC ] || + error_ignore 0000 "stripe QOS didn't balance free space" - rm -rf $DIR/$tdir + simple_cleanup_common } run_test 116a "stripe QOS: free space balance ==================="