if ! echo $PATH | grep -q $LUSTRE/tests; then
export PATH=$LUSTRE/tests:$PATH
fi
+ if ! echo $PATH | grep -q $LUSTRE/../lustre-iokit/sgpdd-survey; then
+ export PATH=$PATH:$LUSTRE/../lustre-iokit/sgpdd-survey
+ fi
export LST=${LST:-"$LUSTRE/../lnet/utils/lst"}
[ ! -f "$LST" ] && export LST=$(which lst)
- export SGPDDSURVEY=${SGPDDSURVEY:-$(which sgpdd-survey)}
+ export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")}
+ [ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey)
# Ubuntu, at least, has a truncate command in /usr/bin
# so fully path our truncate command.
export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate}
export CLIENTMODSONLY=yes
fi
+ export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3}
+
# command line
while getopts "rvwf:" opt $*; do
load_module mgc/mgc
if ! client_only; then
grep -q crc16 /proc/kallsyms || { modprobe crc16 2>/dev/null || true; }
- grep -q jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; }
+ grep -q -w jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; }
+ grep -q -w jbd2 /proc/kallsyms || { modprobe jbd2 2>/dev/null || true; }
[ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs
load_module mgs/mgs
load_module mds/mds
local mntpt=$(facet_mntpt $facet)
echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt"
- do_facet ${facet} mount -t lustre ${!opt} $@ ${!dev} $mntpt
+ do_facet ${facet} "mkdir -p $mntpt; mount -t lustre ${!opt} $@ ${!dev} $mntpt"
RC=${PIPESTATUS[0]}
if [ $RC -ne 0 ]; then
echo "mount -t lustre $@ ${!dev} $mntpt"
shutdown_node_hard () {
local host=$1
- local attempts=3
+ local attempts=$SHUTDOWN_ATTEMPTS
for i in $(seq $attempts) ; do
shutdown_node $host
sleep 1
- ping -w 3 -c 1 $host > /dev/null 2>&1 || return 0
+ wait_for_function --quiet "! ping -w 3 -c 1 $host" 5 1 && return 0
echo "waiting for $host to fail attempts=$attempts"
[ $i -lt $attempts ] || \
{ echo "$host still pingable after power down! attempts=$attempts" && return 1; }
fi
}
-# recovery-scale functions
+facets_hosts () {
+ local facets=$1
+ local hosts
+
+ for facet in ${facets//,/ }; do
+ hosts=$(expand_list $hosts $(facet_host $facet) )
+ done
+
+ echo $hosts
+}
+
+_check_progs_installed () {
+ local progs=$@
+ local rc=0
+
+ for prog in $progs; do
+ if ! [ "$(which $prog)" -o "${!prog}" ]; then
+ echo $prog missing on $(hostname)
+ rc=1
+ fi
+ done
+ return $rc
+}
+
check_progs_installed () {
- local clients=$1
+ local nodes=$1
shift
- local progs=$@
- do_nodes $clients "PATH=:$PATH; status=true;
-for prog in $progs; do
- if ! [ \\\"\\\$(which \\\$prog)\\\" -o \\\"\\\${!prog}\\\" ]; then
- echo \\\$prog missing on \\\$(hostname);
- status=false;
- fi
-done;
-eval \\\$status"
+ do_rpc_nodes $nodes _check_progs_installed $@
}
+# recovery-scale functions
client_var_name() {
echo __$(echo $1 | tr '-' 'X')
}
#######
# General functions
-check_network() {
- local NETWORK=0
- local WAIT=0
- local MAX=$2
- while [ $NETWORK -eq 0 ]; do
- if ping -c 1 -w 3 $1 > /dev/null; then
- NETWORK=1
- else
- WAIT=$((WAIT + 5))
- echo "waiting for $1, $((MAX - WAIT)) secs left"
- sleep 5
- fi
- if [ $WAIT -gt $MAX ]; then
- echo "Network not available"
- exit 1
- fi
+wait_for_function () {
+ local quiet=""
+
+ # suppress fn both stderr and stdout
+ if [ "$1" = "--quiet" ]; then
+ shift
+ quiet=" > /dev/null 2>&1"
+
+ fi
+
+ local fn=$1
+ local max=${2:-900}
+ local sleep=${3:-5}
+
+ local wait=0
+
+ while true; do
+
+ eval $fn $quiet && return 0
+
+ wait=$((wait + sleep))
+ [ $wait -lt $max ] || return 1
+ echo waiting $fn, $((max - wait)) secs left ...
+ sleep $sleep
done
}
-check_port() {
- while( !($DSH2 $1 "netstat -tna | grep -q $2") ) ; do
- sleep 9
- done
+
+check_network() {
+ local host=$1
+ local max=$2
+ local sleep=${3:-5}
+
+ echo `date +"%H:%M:%S (%s)"` waiting for $host network $max secs ...
+ if ! wait_for_function --quiet "ping -c 1 -w 3 $host" $max $sleep ; then
+ echo "Network not available!"
+ exit 1
+ fi
+
+ echo `date +"%H:%M:%S (%s)"` network interface is UP
}
no_dsh() {
(cd `dirname $1`; echo $PWD/`basename $1`)
}
-sync_clients() {
- [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync
- [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync
- cd $SAVE_PWD
-}
check_grant() {
export base=`basetest $1`
[ ${!testname}x == x ] && return 0
echo -n "checking grant......"
- cd $SAVE_PWD
- # write some data to sync client lost_grant
- rm -f $DIR1/${tfile}_check_grant_* 2>&1
- for i in `seq $OSTCOUNT`; do
- $LFS setstripe $DIR1/${tfile}_check_grant_$i -i $(($i -1)) -c 1
- dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
- count=1 > /dev/null 2>&1
- done
- # sync all the data and make sure no pending data on server
- sync_clients
-
- #get client grant and server grant
- client_grant=0
- for d in `lctl get_param -n osc.*.cur_grant_bytes`; do
- client_grant=$((client_grant + $d))
- done
- server_grant=0
- for d in `lctl get_param -n obdfilter.*.tot_granted`; do
- server_grant=$((server_grant + $d))
- done
- # cleanup the check_grant file
- for i in `seq $OSTCOUNT`; do
- rm $DIR1/${tfile}_check_grant_$i
- done
+ local clients=$CLIENTS
+ [ -z $clients ] && clients=$(hostname)
+
+ # sync all the data and make sure no pending data on server
+ do_nodes $clients sync
+
+ # get client grant
+ client_grant=`do_nodes $clients \
+ "$LCTL get_param -n osc.${FSNAME}-*.cur_*grant_bytes" | \
+ awk '{total += $1} END{print total}'`
+
+ # get server grant
+ server_grant=`do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | \
+ awk '{total += $1} END{print total}'`
+
+ # check whether client grant == server grant
+ if [ $client_grant -ne $server_grant ]; then
+ echo "failed: client:${client_grant} server: ${server_grant}."
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL get_param obdfilter.${FSNAME}-OST*.tot*"
+ do_nodes $clients "$LCTL get_param osc.${FSNAME}-*.cur_*_bytes"
+ return 1
+ else
+ echo "pass: client:${client_grant} server: ${server_grant}"
+ fi
- #check whether client grant == server grant
- if [ $client_grant != $server_grant ]; then
- echo "failed: client:${client_grant} server: ${server_grant}"
- return 1
- else
- echo "pass"
- fi
}
########################
git://git.whamcloud.com / fs / lustre-release.git / blobdiff