run_test 412 "mkdir on specific MDTs"
TEST413_COUNT=${TEST413_COUNT:-200}
-generate_uneven_mdts() {
- local threshold=$1
+
+#
+# set_maxage() is used by test_413 only.
+# This is a helper function to set maxage. Does not return any value.
+# Input: maxage to set
+#
+set_maxage() {
local lmv_qos_maxage
local lod_qos_maxage
+ local new_maxage=$1
+
+ lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage)
+ $LCTL set_param lmv.*.qos_maxage=$new_maxage
+ stack_trap "$LCTL set_param \
+ lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null"
+ lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \
+ lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }')
+ do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \
+ lod.*.mdt_qos_maxage=$new_maxage
+ stack_trap "do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \
+ lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null"
+}
+
+generate_uneven_mdts() {
+ local threshold=$1
local ffree
local bavail
local max
local tmp
local i
- lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage)
- $LCTL set_param lmv.*.qos_maxage=1
- stack_trap "$LCTL set_param \
- lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null" RETURN
- lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \
- lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }')
- do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \
- lod.*.mdt_qos_maxage=1
- stack_trap "do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \
- lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" RETURN
-
echo
echo "Check for uneven MDTs: "
fi
done
+ (( min > 0 )) || skip "low space on MDT$min_index"
(( ${ffree[min_index]} > 0 )) ||
- skip "no free files in MDT$min_index"
+ skip "no free files on MDT$min_index"
(( ${ffree[min_index]} < 10000000 )) ||
- skip "too many free files in MDT$min_index"
+ skip "too many free files on MDT$min_index"
# Check if we need to generate uneven MDTs
local diff=$(((max - min) * 100 / min))
- local testdir=$DIR/$tdir-fillmdt
+ local testdirp=$DIR/$tdir-fillmdt # parent fill folder
+ local testdir # individual folder within $testdirp
local start
+ local cmd
- i=0
- while (( diff < threshold )); do
- mkdir -p $testdir
+ # fallocate is faster to consume space on MDT, if available
+ if check_fallocate_supported mds$((min_index + 1)); then
+ cmd="fallocate -l 128K "
+ else
+ cmd="dd if=/dev/zero bs=128K count=1 of="
+ fi
+
+ echo "using cmd $cmd"
+ for (( i = 0; diff < threshold; i++ )); do
+ testdir=${testdirp}/$i
+ [ -d $testdir ] && continue
+
+ (( i % 10 > 0 )) || { $LFS df; $LFS df -i; }
+
+ mkdir -p $testdirp
# generate uneven MDTs, create till $threshold% diff
echo -n "weight diff=$diff% must be > $threshold% ..."
echo "Fill MDT$min_index with $TEST413_COUNT files: loop $i"
- testdir=$DIR/$tdir-fillmdt/$i
- [ -d $testdir ] && continue
$LFS mkdir -i $min_index $testdir ||
error "mkdir $testdir failed"
$LFS setstripe -E 1M -L mdt $testdir ||
error "setstripe $testdir failed"
start=$SECONDS
- for ((F=0; F < TEST413_COUNT; F++)); do
- dd if=/dev/zero of=$testdir/f.$F bs=128K count=1 > \
- /dev/null 2>&1 || error "dd $F failed"
+ for (( f = 0; f < TEST413_COUNT; f++ )); do
+ $cmd$testdir/f.$f &> /dev/null || error "$cmd $f failed"
done
sync; sleep 1; sync
# wait for QOS to update
- (( SECONDS < start + 1 )) && sleep $((start + 1 - SECONDS))
+ (( SECONDS < start + 2 )) && sleep $((start + 2 - SECONDS))
ffree=($(lctl get_param -n mdc.*[mM][dD][cC]-*.filesfree))
bavail=($(lctl get_param -n mdc.*[mM][dD][cC]-*.kbytesavail))
(${bavail[max_index]} * bsize >> 16)))
min=$(((${ffree[min_index]} >> 8) *
(${bavail[min_index]} * bsize >> 16)))
+ (( min > 0 )) || skip "low space on MDT$min_index"
diff=$(((max - min) * 100 / min))
- i=$((i + 1))
done
echo "MDT filesfree available: ${ffree[*]}"
local testdir
local lmv_qos_prio_free
local lmv_qos_threshold_rr
- local lmv_qos_maxage
local lod_qos_prio_free
local lod_qos_threshold_rr
- local lod_qos_maxage
+ local total
local count
local i
+ # @total is total directories created if it's testing plain
+ # directories, otherwise it's total stripe object count for
+ # striped directories test.
+ # remote/striped directory unlinking is slow on zfs and may
+ # timeout, test with fewer directories
+ [ "$mds1_FSTYPE" = "zfs" ] && total=120 || total=240
+
lmv_qos_prio_free=$($LCTL get_param -n lmv.*.qos_prio_free | head -n1)
lmv_qos_prio_free=${lmv_qos_prio_free%%%}
lmv_qos_threshold_rr=$($LCTL get_param -n lmv.*.qos_threshold_rr |
head -n1)
lmv_qos_threshold_rr=${lmv_qos_threshold_rr%%%}
- lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage)
stack_trap "$LCTL set_param \
lmv.*.qos_prio_free=$lmv_qos_prio_free > /dev/null"
stack_trap "$LCTL set_param \
lmv.*.qos_threshold_rr=$lmv_qos_threshold_rr > /dev/null"
- stack_trap "$LCTL set_param \
- lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null"
lod_qos_prio_free=$(do_facet mds1 $LCTL get_param -n \
lod.$FSNAME-MDT0000-mdtlov.mdt_qos_prio_free | head -n1)
lod_qos_threshold_rr=$(do_facet mds1 $LCTL get_param -n \
lod.$FSNAME-MDT0000-mdtlov.mdt_qos_threshold_rr | head -n1)
lod_qos_threshold_rr=${lod_qos_threshold_rr%%%}
- lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \
- lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }')
stack_trap "do_nodes $mdts $LCTL set_param \
lod.*.mdt_qos_prio_free=$lod_qos_prio_free > /dev/null"
stack_trap "do_nodes $mdts $LCTL set_param \
lod.*.mdt_qos_threshold_rr=$lod_qos_threshold_rr > /dev/null"
- stack_trap "do_nodes $mdts $LCTL set_param \
- lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null"
$LCTL set_param lmv.*.qos_threshold_rr=100 > /dev/null
do_nodes $mdts $LCTL set_param lod.*.mdt_qos_threshold_rr=100 > /dev/null
echo "Mkdir (stripe_count $stripe_count) roundrobin:" ||
echo "Mkdir (stripe_count $stripe_count) on stripe $stripe_index"
- stack_trap "unlinkmany -d $testdir/subdir $((100 * MDSCOUNT))"
- for (( i = 0; i < 100 * MDSCOUNT; i++ )); do
+ stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))"
+ for (( i = 0; i < total / stripe_count; i++ )); do
eval $mkdir_cmd $testdir/subdir$i ||
error "$mkdir_cmd subdir$i failed"
done
count=$($LFS getdirstripe -i $testdir/* | grep -c "^$i$")
echo "$count directories created on MDT$i"
if $test_mkdir_rr; then
- (( $count == 100 )) ||
+ (( count == total / stripe_count / MDSCOUNT )) ||
error "subdirs are not evenly distributed"
- elif (( $i == $stripe_index )); then
- (( $count == 100 * MDSCOUNT )) ||
+ elif (( i == stripe_index )); then
+ (( count == total / stripe_count )) ||
error "$count subdirs created on MDT$i"
else
- (( $count == 0 )) ||
+ (( count == 0 )) ||
error "$count subdirs created on MDT$i"
fi
grep -c -P "^\s+$i\t")
echo "$count stripes created on MDT$i"
# deviation should < 5% of average
- (( $count >= 95 * stripe_count &&
- $count <= 105 * stripe_count)) ||
+ delta=$((count - total / MDSCOUNT))
+ (( ${delta#-} <= total / MDSCOUNT / 20 )) ||
error "stripes are not evenly distributed"
fi
done
min_index=$i
fi
done
+ echo "stripe_count=$stripe_count min_idx=$min_index max_idx=$max_index"
- (( ${ffree[min_index]} > 0 )) ||
- skip "no free files in MDT$min_index"
+ (( min > 0 )) || skip "low space on MDT$min_index"
(( ${ffree[min_index]} < 10000000 )) ||
- skip "too many free files in MDT$min_index"
+ skip "too many free files on MDT$min_index"
+
+ generate_uneven_mdts 120
echo "MDT filesfree available: ${ffree[*]}"
echo "MDT blocks available: ${bavail[*]}"
sleep 1
testdir=$DIR/$tdir-s$stripe_count/qos
- local num=200
- stack_trap "unlinkmany -d $testdir/subdir $((num * MDSCOUNT))"
- for (( i = 0; i < num * MDSCOUNT; i++ )); do
+ stack_trap "unlinkmany -d $testdir/subdir $((total / stripe_count))"
+ for (( i = 0; i < total / stripe_count; i++ )); do
eval $mkdir_cmd $testdir/subdir$i ||
error "$mkdir_cmd subdir$i failed"
done
for (( i = 0; i < $MDSCOUNT; i++ )); do
count=$($LFS getdirstripe -i $testdir/* | grep -c "^$i$")
(( count > max )) && max=$count
- echo "$count directories created on MDT$i"
+ echo "$count directories created on MDT$i : curmax=$max"
done
min=$($LFS getdirstripe -i $testdir/* | grep -c "^$min_index$")
- # D-value should > 10% of averge
- (( max - min > num / 10 )) ||
- error "subdirs shouldn't be evenly distributed: $max - $min < $((num / 10))"
+ # D-value should > 10% of average
+ (( max - min > total / stripe_count / MDSCOUNT / 10 )) ||
+ error "subdirs shouldn't be evenly distributed: $max - $min <= $((total / stripe_count / MDSCOUNT / 10))"
# ditto for stripes
if (( stripe_count > 1 )); then
min=$($LFS getdirstripe $testdir/* |
grep -c -P "^\s+$min_index\t")
- (( max - min > num * stripe_count / 10 )) ||
- error "stripes shouldn't be evenly distributed: $max - $min < $((num / 10)) * $stripe_count"
+ (( max - min > total / MDSCOUNT / 10 )) ||
+ error "stripes shouldn't be evenly distributed: $max - $min <= $((total / MDSCOUNT / 10))"
fi
}
[ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
skip "Need server version at least 2.12.52"
+ local stripe_max=$((MDSCOUNT - 1))
local stripe_count
- generate_uneven_mdts 100
- for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do
+ # let caller set maxage for latest result
+ set_maxage 1
+
+ # fill MDT unevenly
+ generate_uneven_mdts 120
+
+ # test 4-stripe directory at most, otherwise it's too slow
+ # We are being very defensive. Although Autotest uses 4 MDTs.
+ # We make sure stripe_max does not go over 4.
+ (( stripe_max > 4 )) && stripe_max=4
+ # unlinking striped directory is slow on zfs, and may timeout, only test
+ # plain directory
+ [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1
+ for stripe_count in $(seq 1 $stripe_max); do
mkdir $DIR/$tdir-s$stripe_count || error "mkdir failed"
mkdir $DIR/$tdir-s$stripe_count/rr || error "mkdir failed"
$LFS mkdir -i $(most_full_mdt) $DIR/$tdir-s$stripe_count/qos ||
[ $MDS1_VERSION -lt $(version_code 2.12.52) ] &&
skip "Need server version at least 2.12.52"
+ local stripe_max=$((MDSCOUNT - 1))
local testdir
local stripe_count
- generate_uneven_mdts 100
- for stripe_count in $(seq 1 $((MDSCOUNT - 1))); do
+ # let caller set maxage for latest result
+ set_maxage 1
+
+ # fill MDT unevenly
+ generate_uneven_mdts 120
+
+ # test 4-stripe directory at most, otherwise it's too slow
+ # We are being very defensive. Although Autotest uses 4 MDTs.
+ # We make sure stripe_max does not go over 4.
+ (( stripe_max > 4 )) && stripe_max=4
+ [ "$mds1_FSTYPE" == "zfs" ] && stripe_max=1
+ for stripe_count in $(seq 1 $stripe_max); do
testdir=$DIR/$tdir-s$stripe_count
mkdir $testdir || error "mkdir $testdir failed"
mkdir $testdir/rr || error "mkdir rr failed"
local testdir
local inherit
local inherit_rr
+ local lmv_qos_maxage
+ local lod_qos_maxage
+
+ # let caller set maxage for latest result
+ lmv_qos_maxage=$($LCTL get_param -n lmv.*.qos_maxage)
+ $LCTL set_param lmv.*.qos_maxage=1
+ stack_trap "$LCTL set_param \
+ lmv.*.qos_maxage=$lmv_qos_maxage > /dev/null" RETURN
+ lod_qos_maxage=$(do_facet mds1 $LCTL get_param -n \
+ lod.$FSNAME-MDT0000-mdtlov.qos_maxage | awk '{ print $1 }')
+ do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \
+ lod.*.mdt_qos_maxage=1
+ stack_trap "do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param \
+ lod.*.mdt_qos_maxage=$lod_qos_maxage > /dev/null" RETURN
+
+ # fill MDT unevenly
+ generate_uneven_mdts 120
testdir=$DIR/${tdir}-s1
mkdir $testdir || error "mkdir $testdir failed"
for pid in $pids; do
wait $pid
done
+
+ true
}
run_test 413z "413 test cleanup"
fi
$LUSTRE/tests/createmany $*
rc=$?
- if (( count > 100 )); then
- debugrestore
- fi
+ debugrestore > /dev/null
return $rc
}
fi
$LUSTRE/tests/unlinkmany $*
rc=$?
- debugrestore
+ debugrestore > /dev/null
return $rc
}
+# Check if fallocate on facet is working. Returns fallocate mode if enabled.
+# Takes optional facet name as argument, to allow separate MDS/OSS checks.
+function check_fallocate_supported()
+{
+ local facet=${1:-ost1}
+ local supported="FALLOCATE_SUPPORTED_$facet"
+ local fstype="${facet}_FSTYPE"
+
+ if [[ -n "${!supported}" ]]; then
+ echo "${!supported}"
+ return 0
+ fi
+ if [[ -z "${!fstype}" ]]; then
+ eval export $fstype=$(facet_fstype $facet)
+ fi
+ if [[ "${!fstype}" != "ldiskfs" ]]; then
+ echo "fallocate on ${!fstype} doesn't consume space" 1>&2
+ return 1
+ fi
+
+ local fa_mode="osd-ldiskfs.$(facet_svc $facet).fallocate_zero_blocks"
+ local mode=$(do_facet $facet $LCTL get_param -n $fa_mode 2>/dev/null |
+ head -n 1)
+
+ if [[ -z "$mode" ]]; then
+ echo "fallocate not supported on $facet" 1>&2
+ return 1
+ fi
+ eval export $supported="$mode"
+
+ echo ${!supported}
+ return 0
+}
+
+# Check if fallocate supported on OSTs, enable if unset, skip if unavailable.
+# Takes optional facet name as argument.
+function check_fallocate_or_skip()
+{
+ local facet=$1
+
+ check_fallocate_supported $1 || skip "fallocate not supported"
+}
+
# Check if fallocate supported on OSTs, enable if unset, default mode=0
# Optionally pass the OST fallocate mode (0=unwritten extents, 1=zero extents)
function check_set_fallocate()
{
local new_mode="$1"
- local osts=$(comma_list $(osts_nodes))
local fa_mode="osd-ldiskfs.*.fallocate_zero_blocks"
- local old_mode=$(do_facet ost1 $LCTL get_param -n $fa_mode 2>/dev/null|
- head -n 1)
+ local old_mode="$(check_fallocate_supported)"
[[ -n "$old_mode" ]] || { echo "fallocate not supported"; return 1; }
[[ -z "$new_mode" && "$old_mode" != "-1" ]] &&
{ echo "keep default fallocate mode: $old_mode"; return 0; }
[[ "$new_mode" && "$old_mode" == "$new_mode" ]] &&
{ echo "keep current fallocate mode: $old_mode"; return 0; }
+ local osts=$(comma_list $(osts_nodes))
stack_trap "do_nodes $osts $LCTL set_param $fa_mode=$old_mode"
do_nodes $osts $LCTL set_param $fa_mode=${new_mode:-0} ||
# Check if fallocate supported on OSTs, enable if unset, skip if unavailable
function check_set_fallocate_or_skip()
{
- [ "$ost1_FSTYPE" != ldiskfs ] && skip "non-ldiskfs backend"
- check_set_fallocate || skip "need at least 2.13.57 for fallocate"
+ check_set_fallocate || skip "need >= 2.13.57 and ldiskfs for fallocate"
}
function disable_opencache()
git://git.whamcloud.com / fs / lustre-release.git / commitdiff