print_summary () {
trap 0
[ "$TESTSUITE" == "lfsck" ] && return 0
- [ -n "$ONLY" ] && echo "WARNING: ONLY is set to ${ONLY}."
- local form="%-13s %-17s %s\n"
- printf "$form" "status" "script" "skipped tests E(xcluded) S(low)"
+ [ -n "$ONLY" ] && echo "WARNING: ONLY is set to $(echo $ONLY)"
+ local details
+ local form="%-13s %-17s %-9s %s %s\n"
+ printf "$form" "status" "script" "Total(sec)" "E(xcluded) S(low)"
echo "------------------------------------------------------------------------------------"
for O in $DEFAULT_SUITES; do
- local skipped=""
- local slow=""
O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]")
- local o=$(echo $O | tr "[:upper:]" "[:lower:]")
- o=${o//_/-}
+ [ "${!O}" = "no" ] && continue || true
+ local o=$(echo $O | tr "[:upper:]_" "[:lower:]-")
local log=${TMP}/${o}.log
- [ -f $log ] && skipped=$(grep excluded $log | awk '{ printf " %s", $3 }' | sed 's/test_//g')
- [ -f $log ] && slow=$(grep SLOW $log | awk '{ printf " %s", $3 }' | sed 's/test_//g')
- [ "${!O}" = "done" ] && \
- printf "$form" "Done" "$O" "E=$skipped" && \
- [ -n "$slow" ] && printf "$form" "-" "-" "S=$slow"
-
+ if is_sanity_benchmark $o; then
+ log=${TMP}/sanity-benchmark.log
+ fi
+ local slow=
+ local skipped=
+ local total=
+ local status=Unfinished
+ if [ -f $log ]; then
+ skipped=$(grep excluded $log | awk '{ printf " %s", $3 }' | sed 's/test_//g')
+ slow=$(egrep "^PASS|^FAIL" $log | tr -d "("| sed s/s\)$//g | sort -nr -k 3 | head -5 | awk '{ print $2":"$3"s" }')
+ total=$(grep duration $log | awk '{ print $2}')
+ if [ "${!O}" = "done" ]; then
+ status=Done
+ fi
+ if $DDETAILS; then
+ local durations=$(egrep "^PASS|^FAIL" $log | tr -d "("| sed s/s\)$//g | awk '{ print $2":"$3"|" }')
+ details=$(printf "%s\n%s %s %s\n" "$details" "DDETAILS" "$O" "$(echo $durations)")
+ fi
+ fi
+ printf "$form" $status "$O" "${total}" "E=$skipped"
+ printf "$form" "-" "-" "-" "S=$(echo $slow)"
done
for O in $DEFAULT_SUITES; do
O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]")
if [ "${!O}" = "no" ]; then
- # FIXME.
- # only for those tests suits which are run directly from acc-sm script:
- # bonnie, iozone, etc.
- if [ -f "$TESTSUITELOG" ] && grep FAIL $TESTSUITELOG | grep -q ' '$O ; then
- printf "$form" "UNFINISHED" "$O" ""
- else
- printf "$form" "Skipped" "$O" ""
- fi
+ printf "$form" "Skipped" "$O" ""
fi
done
- for O in $DEFAULT_SUITES; do
- O=$(echo $O | tr "-" "_" | tr "[:lower:]" "[:upper:]")
- [ "${!O}" = "done" -o "${!O}" = "no" ] || \
- printf "$form" "UNFINISHED" "$O" ""
- done
+ # print the detailed tests durations if DDETAILS=true
+ if $DDETAILS; then
+ echo "$details"
+ fi
}
init_test_env() {
if ! echo $PATH | grep -q $LUSTRE/tests; then
export PATH=$LUSTRE/tests:$PATH
fi
+ if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then
+ export PATH=$PATH:$LUSTRE/../lustre-iokit/sgpdd-survey
+ fi
export LST=${LST:-"$LUSTRE/../lnet/utils/lst"}
[ ! -f "$LST" ] && export LST=$(which lst)
- export SGPDDSURVEY=${SGPDDSURVEY:-$(which sgpdd-survey)}
+ export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")}
+ [ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey)
+ # Ubuntu, at least, has a truncate command in /usr/bin
+ # so fully path our truncate command.
+ export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate}
export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"}
[ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null)
if ! echo $PATH | grep -q $LUSTRE/tests/racer; then
export CLIENTMODSONLY=yes
fi
+ export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3}
+
# command line
while getopts "rvwf:" opt $*; do
shift $((OPTIND - 1))
ONLY=${ONLY:-$*}
+ # print the durations of each test if "true"
+ DDETAILS=${DDETAILS:-false}
[ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
rm -f $TMP/*active
}
eval set -- \$$optvar
if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then
# Nothing in $MODOPTS_<MODULE>; try modprobe.conf
- set -- $(grep "^options\\s*\<${module}\>" $MODPROBECONF)
+ set -- $(grep -P "^options\\s+${BASE}" $MODPROBECONF)
# Get rid of "options $module"
(($# > 0)) && shift 2
load_module mgc/mgc
if ! client_only; then
grep -q crc16 /proc/kallsyms || { modprobe crc16 2>/dev/null || true; }
- grep -q jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; }
+ grep -q -w jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; }
+ grep -q -w jbd2 /proc/kallsyms || { modprobe jbd2 2>/dev/null || true; }
[ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs
load_module mgs/mgs
load_module mds/mds
}
# Facet functions
+mount_facets () {
+ local facets=${1:-$(get_facets)}
+ local facet
+
+ for facet in ${facets//,/ }; do
+ mount_facet $facet || error "Restart of $facet failed!"
+ done
+}
+
mount_facet() {
local facet=$1
shift
local mntpt=$(facet_mntpt $facet)
echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt"
- do_facet ${facet} mount -t lustre ${!opt} $@ ${!dev} $mntpt
+ do_facet ${facet} "mkdir -p $mntpt; mount -t lustre ${!opt} $@ ${!dev} $mntpt"
RC=${PIPESTATUS[0]}
if [ $RC -ne 0 ]; then
echo "mount -t lustre $@ ${!dev} $mntpt"
}
zconf_mount() {
- local OPTIONS
local client=$1
local mnt=$2
- # Only supply -o to mount if we have options
- if [ -n "$MOUNTOPT" ]; then
- OPTIONS="-o $MOUNTOPT"
- fi
+ local OPTIONS=${3:-$MOUNTOPT}
+
local device=$MGSNID:/$FSNAME
if [ -z "$mnt" -o -z "$FSNAME" ]; then
echo Bad zconf mount command: opt=$OPTIONS dev=$device mnt=$mnt
local mnt=$2
local OPTIONS=${3:-$MOUNTOPT}
- # Only supply -o to mount if we have options
- if [ "$OPTIONS" ]; then
- OPTIONS="-o $OPTIONS"
- fi
local device=$MGSNID:/$FSNAME
if [ -z "$mnt" -o -z "$FSNAME" ]; then
echo Bad zconf mount command: opt=$OPTIONS dev=$device mnt=$mnt
shutdown_node_hard () {
local host=$1
- local attempts=3
+ local attempts=$SHUTDOWN_ATTEMPTS
for i in $(seq $attempts) ; do
shutdown_node $host
sleep 1
- ping -w 3 -c 1 $host > /dev/null 2>&1 || return 0
+ wait_for_function --quiet "! ping -w 3 -c 1 $host" 5 1 && return 0
echo "waiting for $host to fail attempts=$attempts"
[ $i -lt $attempts ] || \
{ echo "$host still pingable after power down! attempts=$attempts" && return 1; }
echo $(comma_list $affected)
}
+facet_up () {
+ local facet=$1
+ local host=${2:-$(facet_host $facet)}
+
+ local label=$(convert_facet2label $facet)
+ do_node $host lctl dl | awk '{print $4}' | grep -q $label
+}
+
+facets_up_on_host () {
+ local host=$1
+ local facets=$(facets_on_host $host)
+ local affected_up
+
+ for facet in ${facets//,/ }; do
+ if $(facet_up $facet $host); then
+ affected_up="$affected_up $facet"
+ fi
+ done
+
+ echo $(comma_list $affected_up)
+}
+
shutdown_facet() {
local facet=$1
fi
}
-# recovery-scale functions
+facets_hosts () {
+ local facets=$1
+ local hosts
+
+ for facet in ${facets//,/ }; do
+ hosts=$(expand_list $hosts $(facet_host $facet) )
+ done
+
+ echo $hosts
+}
+
+_check_progs_installed () {
+ local progs=$@
+ local rc=0
+
+ for prog in $progs; do
+ if ! [ "$(which $prog)" -o "${!prog}" ]; then
+ echo $prog missing on $(hostname)
+ rc=1
+ fi
+ done
+ return $rc
+}
+
check_progs_installed () {
- local clients=$1
+ local nodes=$1
shift
- local progs=$@
- do_nodes $clients "PATH=:$PATH; status=true;
-for prog in $progs; do
- if ! [ \\\"\\\$(which \\\$prog)\\\" -o \\\"\\\${!prog}\\\" ]; then
- echo \\\$prog missing on \\\$(hostname);
- status=false;
- fi
-done;
-eval \\\$status"
+ do_rpc_nodes $nodes _check_progs_installed $@
}
+# recovery-scale functions
client_var_name() {
echo __$(echo $1 | tr '-' 'X')
}
rm $MOUNT/recon
}
+affected_facets () {
+ local facet=$1
+
+ local host=$(facet_active_host $facet)
+ local affected=$facet
+
+ if [ "$FAILURE_MODE" = HARD ]; then
+ affected=$(facets_up_on_host $host)
+ fi
+ echo $affected
+}
+
facet_failover() {
local facet=$1
local sleep_time=$2
echo "Failing $facet on node $host"
- local affected=$facet
-
- if [ "$FAILURE_MODE" = HARD ]; then
- affected=$(facets_on_host $host)
- fi
+ local affected=$(affected_facets $facet)
shutdown_facet $facet
fi
# FIXME; has to be changed to mount all facets concurrently
affected=$(exclude_items_from_list $affected mgs)
- for facet in ${affected//,/ }; do
- mount_facet $facet || error "Restart of $facet on node $host failed!"
- done
+ mount_facets $affected
}
obd_name() {
local facet=$1
do_facet $facet sync
df $MOUNT
+
+ # make sure there will be no seq change
+ local clients=${CLIENTS:-$HOSTNAME}
+ do_nodes $clients "f=${MOUNT}/fsa-\\\$(hostname); mcreate \\\$f; rm \\\$f"
+
local svc=${facet}_svc
do_facet $facet $LCTL --device %${!svc} notransno
do_facet $facet $LCTL --device %${!svc} readonly
#######
# General functions
-check_network() {
- local NETWORK=0
- local WAIT=0
- local MAX=$2
- while [ $NETWORK -eq 0 ]; do
- if ping -c 1 -w 3 $1 > /dev/null; then
- NETWORK=1
- else
- WAIT=$((WAIT + 5))
- echo "waiting for $1, $((MAX - WAIT)) secs left"
- sleep 5
- fi
- if [ $WAIT -gt $MAX ]; then
- echo "Network not available"
- exit 1
- fi
+wait_for_function () {
+ local quiet=""
+
+ # suppress fn both stderr and stdout
+ if [ "$1" = "--quiet" ]; then
+ shift
+ quiet=" > /dev/null 2>&1"
+
+ fi
+
+ local fn=$1
+ local max=${2:-900}
+ local sleep=${3:-5}
+
+ local wait=0
+
+ while true; do
+
+ eval $fn $quiet && return 0
+
+ wait=$((wait + sleep))
+ [ $wait -lt $max ] || return 1
+ echo waiting $fn, $((max - wait)) secs left ...
+ sleep $sleep
done
}
-check_port() {
- while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do
- sleep 9
- done
+
+check_network() {
+ local host=$1
+ local max=$2
+ local sleep=${3:-5}
+
+ echo `date +"%H:%M:%S (%s)"` waiting for $host network $max secs ...
+ if ! wait_for_function --quiet "ping -c 1 -w 3 $host" $max $sleep ; then
+ echo "Network not available!"
+ exit 1
+ fi
+
+ echo `date +"%H:%M:%S (%s)"` network interface is UP
}
no_dsh() {
echo "$@" > $LOGDIR/err
}
+exit_status () {
+ local status=0
+ local log=$TESTSUITELOG
+
+ [ -f "$log" ] && grep -q FAIL $log && status=1
+ exit $status
+}
+
error() {
error_noexit "$@"
exit 1
return 1
}
+complete () {
+ equals_msg $1 test complete, duration $2 sec
+ [ -f "$TESTSUITELOG" ] && egrep .FAIL $TESTSUITELOG || true
+ echo duration $2 >>$TESTSUITELOG
+}
+
pass() {
# Set TEST_STATUS here; will be used for logging the result
if [ -f $LOGDIR/err ]; then
else
TEST_STATUS="PASS"
fi
- echo $TEST_STATUS " " $@
+ echo "$TEST_STATUS $@" 2>&1 | tee -a $TESTSUITELOG
}
check_mds() {
echo "test_$1 returned $RC" | tee $LOGDIR/err
duration=$((`date +%s` - $BEFORE))
- pass "(${duration}s)"
+ pass "$1" "(${duration}s)"
[ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
(cd `dirname $1`; echo $PWD/`basename $1`)
}
-sync_clients() {
- [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync
- [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync
- cd $SAVE_PWD
-}
check_grant() {
export base=`basetest $1`
[ ${!testname}x == x ] && return 0
echo -n "checking grant......"
- cd $SAVE_PWD
- # write some data to sync client lost_grant
- rm -f $DIR1/${tfile}_check_grant_* 2>&1
- for i in `seq $OSTCOUNT`; do
- $LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1
- dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
- count=1 > /dev/null 2>&1
- done
- # sync all the data and make sure no pending data on server
- sync_clients
-
- #get client grant and server grant
- client_grant=0
- for d in `lctl get_param -n osc.*.cur_grant_bytes`; do
- client_grant=$((client_grant + $d))
- done
- server_grant=0
- for d in `lctl get_param -n obdfilter.*.tot_granted`; do
- server_grant=$((server_grant + $d))
- done
- # cleanup the check_grant file
- for i in `seq $OSTCOUNT`; do
- rm $DIR1/${tfile}_check_grant_$i
- done
+ local clients=$CLIENTS
+ [ -z $clients ] && clients=$(hostname)
+
+ # sync all the data and make sure no pending data on server
+ do_nodes $clients sync
+
+ # get client grant
+ client_grant=`do_nodes $clients \
+ "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" | \
+ awk '{total += $1} END{print total}'`
+
+ # get server grant
+ server_grant=`do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | \
+ awk '{total += $1} END{print total}'`
+
+ # check whether client grant == server grant
+ if [ $client_grant -ne $server_grant ]; then
+ echo "failed: client:${client_grant} server: ${server_grant}."
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*"
+ do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes"
+ return 1
+ else
+ echo "pass: client:${client_grant} server: ${server_grant}"
+ fi
- #check whether client grant == server grant
- if [ $client_grant != $server_grant ]; then
- echo "failed: client:${client_grant} server: ${server_grant}"
- return 1
- else
- echo "pass"
- fi
}
########################
cat ${rslt}.detail
}
+# returns the canonical name for an ldiskfs device
+ldiskfs_canon() {
+ local dev="$1"
+ local facet="$2"
+
+ do_facet $facet "dv=\\\$(lctl get_param -n $dev);
+if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then
+ echo dm-\\\${foo##*:};
+else
+ echo \\\$(basename \\\$dv);
+fi;"
+}
+
+is_sanity_benchmark() {
+ local benchmarks="dbench bonnie iozone fsx"
+ local suite=$1
+ for b in $benchmarks; do
+ if [ "$b" == "$suite" ]; then
+ return 0
+ fi
+ done
+ return 1
+}
+
+min_ost_size () {
+ $LCTL get_param -n osc.*.kbytesavail | sort -n | head -n1
+}
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff