set -e
+export REFORMAT=""
+export VERBOSE=false
+
# eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS
assert_env() {
local failed=""
init_test_env() {
export LUSTRE=`absolute_path $LUSTRE`
export TESTSUITE=`basename $0 .sh`
- export XMLCONFIG="${TESTSUITE}.xml"
+ export XMLCONFIG=${XMLCONFIG:-${TESTSUITE}.xml}
export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest}
[ -d /r ] && export ROOT=/r
+ export TMP=${TMP:-$ROOT/tmp}
export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
+ export LLMOUNT=${LLMOUNT:-"llmount"}
export LCONF=${LCONF:-"lconf"}
export LMC=${LMC:-"lmc"}
- export LCTL=${LCTL:-"lctl"}
- export CHECKSTAT="${CHECKSTAT:-checkstat} -v"
+ export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
+ export CHECKSTAT="${CHECKSTAT:-checkstat} "
+ export FSYTPE=${FSTYPE:-"ext3"}
# Paths on remote nodes, if different
export RLUSTRE=${RLUSTRE:-$LUSTRE}
# command line
- while getopts "rf:" opt $*; do
+ while getopts "rvf:" opt $*; do
case $opt in
f) CONFIG=$OPTARG;;
r) REFORMAT=--reformat;;
+ v) VERBOSE=true;;
\?) usage;;
esac
done
# save the name of the config file for the upcall
echo "XMLCONFIG=$LUSTRE/tests/$XMLCONFIG" > $LUSTRE/tests/XMLCONFIG
+# echo "CONFIG=`canonical_path $CONFIG`" > $LUSTRE/tests/CONFIG
}
# Facet functions
facet=$1
shift
active=`facet_active $facet`
- do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ $XMLCONFIG
+ do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+ --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+ $@ $XMLCONFIG
}
stop() {
facet=$1
active=`facet_active $facet`
shift
- do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ --cleanup $XMLCONFIG
+ do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \
+ --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \
+ $@ --cleanup $XMLCONFIG
}
zconf_mount() {
- mnt=$1
+ client=$1
+ mnt=$2
+
+ do_node $client mkdir $mnt 2> /dev/null || :
- [ -d $mnt ] || mkdir $mnt
-
if [ -x /sbin/mount.lustre ] ; then
- mount -t lustre -o nettype=$NETTYPE \
- `facet_host mds`:/mds_svc/client_facet $mnt
+ do_node $client mount -t lustre -o nettype=$NETTYPE `facet_active_host mds1`:/mds1_svc/client_facet $mnt || return 1
else
# this is so cheating
+ do_node $client $LCONF --nosetup --node client_facet $XMLCONFIG > /dev/null || return 2
$LCONF --nosetup --node client_facet $XMLCONFIG
- $LUSTRE/utils/llmount `facet_host mds`:/mds_svc/client_facet $mnt \
- -o nettype=$NETTYPE
+ do_node $client $LLMOUNT `facet_active_host mds1`:/mds1_svc/client_facet $mnt -o nettype=$NETTYPE|| return 4
fi
[ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname`
}
zconf_umount() {
- mnt=$1
- umount $mnt || :
- $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG || :
+ client=$1
+ mnt=$2
+ [ "$3" ] && force=-f
+ do_node $client umount $force $mnt || :
+ do_node $client $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG > /dev/null || :
}
shutdown_facet() {
facet=$1
if [ "$FAILURE_MODE" = HARD ]; then
$POWER_DOWN `facet_active_host $facet`
+ sleep 2
elif [ "$FAILURE_MODE" = SOFT ]; then
stop $facet --force --failover --nomod
fi
wait_for_host() {
HOST=$1
check_network $HOST 900
- while ! $PDSH $HOST "ls -ld $LUSTRE"; do sleep 5; done
+ while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done
}
wait_for() {
client_df() {
# not every config has many clients
if [ ! -z "$CLIENTS" ]; then
- $PDSH $CLIENTS "df $MOUNT" | dshbak -c
+ $PDSH $CLIENTS "df $MOUNT" > /dev/null
fi
}
facet=$1
echo "Failing $facet node `facet_active_host $facet`"
shutdown_facet $facet
- sleep 2
reboot_facet $facet
client_df &
DFPID=$!
+ echo "df pid is $DFPID"
change_active $facet
TO=`facet_active_host $facet`
- echo "Failover MDS to $TO"
+ echo "Failover $facet to $TO"
wait_for $facet
start $facet
}
mds_evict_client() {
UUID=`cat /proc/fs/lustre/mdc/*_MNT_*/uuid`
- do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds_svc/evict_client"
+ do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds1_svc/evict_client"
}
fail() {
$LMC -m ${XMLCONFIG} $@
}
+h2gm () {
+ if [ "$1" = "client" ]; then echo \'*\'; else
+ $PDSH $1 $GMNALNID -l | cut -d\ -f2
+ fi
+}
+
h2tcp() {
if [ "$1" = "client" ]; then echo \'*\'; else
echo $1
facet_active_host() {
local facet=$1
local active=`facet_active $facet`
- echo `facet_host $active`
+ if [ "$facet" == client ]; then
+ hostname
+ else
+ echo `facet_host $active`
+ fi
}
change_active() {
do_node() {
HOST=$1
shift
+
+ if $VERBOSE; then
+ echo "CMD: $HOST $@"
+ $PDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || :
+ fi
$PDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; sh -c \"$@\")"
}
do_facet() {
facet=$1
shift
+
+ if [ "$facet" == "mds" ]; then
+ if [ "$MDSCOUNT" -gt 1 ]; then
+ for num in `seq $MDSCOUNT`; do
+ HOST=`facet_active_host $facet$num`
+ do_node $HOST $@
+ done
+ return
+ fi
+ fi
HOST=`facet_active_host $facet`
do_node $HOST $@
}
local facet=$1
shift
echo "add facet $facet: `facet_host $facet`"
- do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT
+ do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT \
+ --lustre_upcall $UPCALL --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM
do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \
--nettype $NETTYPE
}
facet=$1
shift
rm -f ${facet}active
- add_facet $facet --lustre_upcall $UPCALL
- do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc $*
+ add_facet $facet
+ do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc --fstype $FSTYPE $*
}
add_mdsfailover() {
facet=$1
shift
add_facet ${facet}failover --lustre_upcall $UPCALL
- do_lmc --add mds --node ${facet}failover_facet --mds ${facet}_svc $*
+ do_lmc --add mds --node ${facet}failover_facet --mds ${facet}_svc --fstype $FSTYPE $*
}
add_ost() {
shift
rm -f ${facet}active
add_facet $facet
- do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc $*
+ do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc --fstype $FSTYPE $*
+}
+
+del_ost() {
+ facet=$1
+ shift
+ do_lmc --delete ost --node ${facet}_facet --ost ${facet}_svc $*
+}
+
+deactivate_ost() {
+ facet=$1
+ shift
+ do_lmc --deactivate ost --node ${facet}_facet --ost ${facet}_svc $*
}
add_ostfailover() {
facet=$1
shift
add_facet ${facet}failover
- do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc $*
+ do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc --fstype $FSTYPE $*
}
add_lov() {
mds_facet=$2
shift; shift
do_lmc --add lov --mds ${mds_facet}_svc --lov $lov $*
-
+}
+
+add_lov_to_lmv() {
+ lov=$1
+ lmv=$2
+ shift; shift
+ do_lmc --add lov --lmv $lmv --lov $lov $*
+}
+
+add_lmv() {
+ lmv=$1
+ shift;
+ do_lmc --add lmv --lmv $lmv $*
}
add_client() {
facet=$1
- mds=$2
- shift; shift
+ shift;
add_facet $facet --lustre_upcall $UPCALL
- do_lmc --add mtpt --node ${facet}_facet --mds ${mds}_svc $*
-
+ do_lmc --add mtpt --node ${facet}_facet $*
}
+config_commit() {
+ do_lmc --commit
+}
#######
# General functions
}
##################################
+# OBD_FAIL funcs
+
+drop_request() {
+# OBD_FAIL_MDS_ALL_REQUEST_NET
+ RC=0
+ do_facet mds "echo 0x123 > /proc/sys/lustre/fail_loc"
+ do_facet client "$1" || RC=$?
+ do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+drop_reply() {
+# OBD_FAIL_MDS_ALL_REPLY_NET
+ RC=0
+ do_facet mds "echo 0x122 > /proc/sys/lustre/fail_loc"
+ do_facet client "$@" || RC=$?
+ do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+drop_reint_reply() {
+# OBD_FAIL_MDS_REINT_NET_REP
+ RC=0
+ do_facet mds "echo 0x119 > /proc/sys/lustre/fail_loc"
+ do_facet client "$@" || RC=$?
+ do_facet mds "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+pause_bulk() {
+#define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214
+ RC=0
+ do_facet ost "echo 0x214 > /proc/sys/lustre/fail_loc"
+ do_facet client "$1" || RC=$?
+ do_facet client "sync"
+ do_facet ost "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+drop_ldlm_cancel() {
+#define OBD_FAIL_LDLM_CANCEL 0x304
+ RC=0
+ do_facet client "echo 0x304 > /proc/sys/lustre/fail_loc"
+ do_facet client "$@" || RC=$?
+ do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+drop_bl_callback() {
+#define OBD_FAIL_LDLM_BL_CALLBACK 0x305
+ RC=0
+ do_facet client "echo 0x305 > /proc/sys/lustre/fail_loc"
+ do_facet client "$@" || RC=$?
+ do_facet client "echo 0 > /proc/sys/lustre/fail_loc"
+ return $RC
+}
+
+clear_failloc() {
+ facet=$1
+ pause=$2
+ sleep $pause
+ echo "clearing fail_loc on $facet"
+ do_facet $facet "sysctl -w lustre.fail_loc=0"
+}
+
+cancel_lru_locks() {
+ $LCTL mark cancel_lru_locks
+ for d in /proc/fs/lustre/ldlm/namespaces/$1*; do
+ if [ -f $d/lru_size ]; then
+ echo clear > $d/lru_size
+ grep [0-9] $d/lock_unused_count
+ fi
+ done
+}
+
+
+pgcache_empty() {
+ for a in /proc/fs/lustre/llite/*/dump_page_cache; do
+ if [ `wc -l $a | awk '{print $1}'` -gt 1 ]; then
+ echo there is still data in page cache $a ?
+ cat $a;
+ return 1;
+ fi
+ done
+ return 0
+}
+
+##################################
# Test interface
error() {
echo "${TESTSUITE}: **** FAIL:" $@
+ log "FAIL: $@"
exit 1
}
printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS
}
+log() {
+ echo "$*"
+ lctl mark "$*" 2> /dev/null || true
+}
+
run_one() {
testnum=$1
message=$2
- tfile=f$base
- tdir=d$base
+ tfile=f${testnum}
+ tdir=d${base}
# Pretty tests run faster.
equals_msg $testnum: $message
+ log "== test $1: $2"
test_${testnum} || error "test_$testnum failed with $?"
}
+
+canonical_path() {
+ (cd `dirname $1`; echo $PWD/`basename $1`)
+}
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff