X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=c3bf14b7bf75b47536958b8f6986c64a693696dd;hb=16dcdf3b09ee15eb8bd5b43cbe413dff72bf3bca;hp=01150a075a4cca267d073c23b54701b5693fed8f;hpb=c5050e412572b00cbe93d8517d2d1f767bebfa92;p=fs%2Flustre-release.git diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 01150a0..c3bf14b 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1,6 +1,8 @@ #!/bin/sh set -e +export REFORMAT="" +export VERBOSE=false # eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS assert_env() { @@ -24,16 +26,25 @@ usage() { init_test_env() { export LUSTRE=`absolute_path $LUSTRE` export TESTSUITE=`basename $0 .sh` - export XMLCONFIG="${TESTSUITE}.xml" + export XMLCONFIG=${XMLCONFIG:-${TESTSUITE}.xml} export LTESTDIR=${LTESTDIR:-$LUSTRE/../ltest} - [ -d /r ] && export ROOT=/r + [ -d /r ] && test "x$ROOT" = "x" && export ROOT=/r + export TMP=${TMP:-$ROOT/tmp} export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests + export LLMOUNT=${LLMOUNT:-"llmount"} export LCONF=${LCONF:-"lconf"} export LMC=${LMC:-"lmc"} - export LCTL=${LCTL:-"lctl"} - export CHECKSTAT="${CHECKSTAT:-checkstat} -v" + export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} + export CHECKSTAT="${CHECKSTAT:-checkstat} " + DEF_FSTYPE=`test "x$(uname -r | grep -o '2.6')" = "x2.6" && echo "ldiskfs" || echo "ext3"` + export FSTYPE=${FSTYPE:-$DEF_FSTYPE} + #used only if FSTYPE == smfs, otherwise ignored by lconf + MDS_BACKFSTYPE=${MDS_BACKFSTYPE:-$DEF_FSTYPE} + OST_BACKFSTYPE=${OST_BACKFSTYPE:-$DEF_FSTYPE} + + export SECURITY=${SECURITY:-"null"} # Paths on remote nodes, if different export RLUSTRE=${RLUSTRE:-$LUSTRE} @@ -41,46 +52,58 @@ init_test_env() { # command line - while getopts "rf:" opt $*; do + while getopts "rvf:" opt $*; do case $opt in f) CONFIG=$OPTARG;; r) REFORMAT=--reformat;; + v) VERBOSE=true;; \?) usage;; esac done + shift $((OPTIND - 1)) + ONLY=${ONLY:-$*} + # save the name of the config file for the upcall echo "XMLCONFIG=$LUSTRE/tests/$XMLCONFIG" > $LUSTRE/tests/XMLCONFIG +# echo "CONFIG=`canonical_path $CONFIG`" > $LUSTRE/tests/CONFIG } +. krb5_env.sh + # Facet functions start() { facet=$1 shift active=`facet_active $facet` - do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ $XMLCONFIG + do_facet $facet $LCONF --select ${facet}_svc=${active}_facet -v \ + --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \ + --mds_sec $SECURITY $@ $XMLCONFIG } stop() { facet=$1 active=`facet_active $facet` shift - do_facet $facet $LCONF --select ${facet}_svc=${active}_facet --node ${active}_facet --ptldebug $PTLDEBUG $@ --cleanup $XMLCONFIG + do_facet $facet $LCONF --select ${facet}_svc=${active}_facet \ + --node ${active}_facet --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM \ + $@ --cleanup $XMLCONFIG } zconf_mount() { - mnt=$1 + client=$1 + mnt=$2 + + do_node $client mkdir $mnt 2> /dev/null || : - [ -d $mnt ] || mkdir $mnt - if [ -x /sbin/mount.lustre ] ; then - mount -t lustre -o nettype=$NETTYPE \ - `facet_host mds`:/mds_svc/client_facet $mnt + do_node $client mount -t lustre -o mds_sec=$SECURITY,nettype=$NETTYPE \ + `facet_active_host mds1`:/mds1_svc/client_facet $mnt || return 2 else # this is so cheating - $LCONF --nosetup --node client_facet $XMLCONFIG - $LUSTRE/utils/llmount `facet_host mds`:/mds_svc/client_facet $mnt \ - -o nettype=$NETTYPE + do_node $client $LCONF --nosetup --node client_facet $XMLCONFIG > /dev/null || return 2 + do_node $client $LLMOUNT `facet_active_host mds1`:/mds1_svc/client_facet $mnt \ + -o mds_sec=$SECURITY,nettype=$NETTYPE|| return 4 fi [ -d /r ] && $LCTL modules > /r/tmp/ogdb-`hostname` @@ -88,15 +111,18 @@ zconf_mount() { } zconf_umount() { - mnt=$1 - umount $mnt || : - $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG || : + client=$1 + mnt=$2 + [ "$3" ] && force=-f + do_node $client umount $force $mnt || : + do_node $client $LCONF --cleanup --nosetup --node client_facet $XMLCONFIG > /dev/null || : } shutdown_facet() { facet=$1 if [ "$FAILURE_MODE" = HARD ]; then $POWER_DOWN `facet_active_host $facet` + sleep 2 elif [ "$FAILURE_MODE" = SOFT ]; then stop $facet --force --failover --nomod fi @@ -112,7 +138,7 @@ reboot_facet() { wait_for_host() { HOST=$1 check_network $HOST 900 - while ! $PDSH $HOST "ls -ld $LUSTRE"; do sleep 5; done + while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done } wait_for() { @@ -124,7 +150,7 @@ wait_for() { client_df() { # not every config has many clients if [ ! -z "$CLIENTS" ]; then - $PDSH $CLIENTS "df $MOUNT" | dshbak -c + $PDSH $CLIENTS "df $MOUNT" > /dev/null fi } @@ -132,13 +158,13 @@ facet_failover() { facet=$1 echo "Failing $facet node `facet_active_host $facet`" shutdown_facet $facet - sleep 2 reboot_facet $facet client_df & DFPID=$! + echo "df pid is $DFPID" change_active $facet TO=`facet_active_host $facet` - echo "Failover MDS to $TO" + echo "Failover $facet to $TO" wait_for $facet start $facet } @@ -153,9 +179,18 @@ replay_barrier() { $LCTL mark "REPLAY BARRIER" } +replay_barrier_nodf() { + local facet=$1 + do_facet $facet sync + do_facet $facet $LCTL --device %${facet}_svc readonly + do_facet $facet $LCTL --device %${facet}_svc notransno + do_facet $facet $LCTL mark "REPLAY BARRIER" + $LCTL mark "REPLAY BARRIER" +} + mds_evict_client() { UUID=`cat /proc/fs/lustre/mdc/*_MNT_*/uuid` - do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds_svc/evict_client" + do_facet mds "echo $UUID > /proc/fs/lustre/mds/mds1_svc/evict_client" } fail() { @@ -164,6 +199,15 @@ fail() { df $MOUNT || error "post-failover df: $?" } +fail_drop() { + local facet=$1 + local failcode=$2 + facet_failover $facet + do_facet mds "echo $failcode > /proc/sys/lustre/fail_loc" + df $MOUNT || error "post-failover df: $?" + do_facet mds "echo 0 > /proc/sys/lustre/fail_loc" +} + fail_abort() { local facet=$1 stop $facet --force --failover --nomod @@ -171,6 +215,7 @@ fail_abort() { start $facet do_facet $facet lctl --device %${facet}_svc abort_recovery df $MOUNT || echo "first df failed: $?" + sleep 1 df $MOUNT || error "post-failover df: $?" } @@ -178,20 +223,34 @@ do_lmc() { $LMC -m ${XMLCONFIG} $@ } +h2gm () { + if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else + $PDSH $1 $GMNALNID -l | cut -d\ -f2 + fi +} + h2tcp() { - if [ "$1" = "client" ]; then echo \'*\'; else + if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else echo $1 fi } declare -fx h2tcp h2elan() { - if [ "$1" = "client" ]; then echo \'*\'; else + if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else echo $1 | sed 's/[^0-9]*//g' fi } declare -fx h2elan +h2openib() { + if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else + echo $1 | sed 's/[^0-9]*//g' + fi +} +declare -fx h2openib + + facet_host() { local facet=$1 varname=${facet}_HOST @@ -211,6 +270,11 @@ facet_nid() { facet_active() { local facet=$1 local activevar=${facet}active + + if [ -f ./${facet}active ] ; then + source ./${facet}active + fi + active=${!activevar} if [ -z "$active" ] ; then echo -n ${facet} @@ -222,7 +286,11 @@ facet_active() { facet_active_host() { local facet=$1 local active=`facet_active $facet` - echo `facet_host $active` + if [ "$facet" == client ]; then + hostname + else + echo `facet_host $active` + fi } change_active() { @@ -244,20 +312,39 @@ change_active() { do_node() { HOST=$1 shift - $PDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; cd $RPWD; sh -c \"$@\")" + + if $VERBOSE; then + echo "CMD: $HOST $@" + $PDSH $HOST $LCTL mark "$@" > /dev/null 2>&1 || : + fi + $PDSH $HOST "(PATH=$RLUSTRE/utils:$RLUSTRE/tests:/usr/sbin:/sbin:\$PATH; cd $RPWD; sh -c \"$@\")" +} + +mds_list() { + seq -f mds%g $MDSCOUNT } + do_facet() { facet=$1 shift - HOST=`facet_active_host $facet` - do_node $HOST $@ + + if [ "$facet" == "mds" ]; then + for mds in `mds_list`; do + HOST=`facet_active_host $mds` + do_node $HOST $@ + done + else + HOST=`facet_active_host $facet` + do_node $HOST $@ + fi } add_facet() { local facet=$1 shift echo "add facet $facet: `facet_host $facet`" - do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT + do_lmc --add node --node ${facet}_facet $@ --timeout $TIMEOUT \ + --lustre_upcall $UPCALL --ptldebug $PTLDEBUG --subsystem $SUBSYSTEM do_lmc --add net --node ${facet}_facet --nid `facet_nid $facet` \ --nettype $NETTYPE } @@ -266,15 +353,15 @@ add_mds() { facet=$1 shift rm -f ${facet}active - add_facet $facet --lustre_upcall $UPCALL - do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc $* + add_facet $facet + do_lmc --add mds --node ${facet}_facet --mds ${facet}_svc --fstype $FSTYPE --backfstype $MDS_BACKFSTYPE $* } add_mdsfailover() { facet=$1 shift add_facet ${facet}failover --lustre_upcall $UPCALL - do_lmc --add mds --node ${facet}failover_facet --mds ${facet}_svc $* + do_lmc --add mds --node ${facet}failover_facet --mds ${facet}_svc --fstype $FSTYPE --backfstype $MDS_BACKFSTYPE $* } add_ost() { @@ -282,14 +369,61 @@ add_ost() { shift rm -f ${facet}active add_facet $facet - do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc $* + do_lmc --add ost --node ${facet}_facet --ost ${facet}_svc --fstype $FSTYPE --backfstype $OST_BACKFSTYPE $* +} + +del_ost() { + facet=$1 + shift + do_lmc --delete ost --node ${facet}_facet --ost ${facet}_svc $* +} +start_gks() { + facet=$1 + shift + rm -f ${facet}active + add_facet $facet + do_facet $facet $LCONF --node ${facet}_facet --ptldebug $PTLDEBUG $* $XMLCONFIG +} +stop_gks() { + facet=$1 + shift + do_facet $facet $LCONF --node ${facet}_facet --cleanup $* $XMLCONFIG +} + +add_gks() { + facet=$1 + shift + rm -f ${facet}active + add_facet $facet + do_lmc --add gks --gks ${facet}_svc --node ${facet}_facet $* +} +add_cmobd() { + facet=$1 + cache_facet=$2 + master_facet=$3 + shift; shift; shift + do_lmc --add cmobd --node mds1_facet --cmobd ${facet}_svc --cache_obd ${cache_facet} --master_obd ${master_facet} +} + +add_cobd() { + facet=$1 + client_facet=$2 + master_facet=$3 + shift; shift; shift + do_lmc --add cobd --node mds1_facet --cmobd ${facet}_svc --cache_obd ${cache_facet} --master_obd ${master_facet} +} + +deactivate_ost() { + facet=$1 + shift + do_lmc --deactivate ost --node ${facet}_facet --ost ${facet}_svc $* } add_ostfailover() { facet=$1 shift add_facet ${facet}failover - do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc $* + do_lmc --add ost --failover --node ${facet}failover_facet --ost ${facet}_svc --fstype $FSTYPE --backfstype $OST_BACKFSTYPE $* } add_lov() { @@ -297,7 +431,27 @@ add_lov() { mds_facet=$2 shift; shift do_lmc --add lov --mds ${mds_facet}_svc --lov $lov $* - +} + +add_lov_to_lmv() { + lov=$1 + lmv=$2 + shift; shift + do_lmc --add lov --lmv $lmv --lov $lov $* +} + +add_lov_to_cache_master_lmv() { + lov=$1 + cache_lmv=$2 + master_lmv=$3 + shift; shift; shift + do_lmc --add lov --cachelmv $cache_lmv --masterlmv $master_lmv --lov $lov $* +} + +add_lmv() { + lmv=$1 + shift; + do_lmc --add lmv --lmv $lmv $* } add_client() { @@ -305,10 +459,12 @@ add_client() { mds=$2 shift; shift add_facet $facet --lustre_upcall $UPCALL - do_lmc --add mtpt --node ${facet}_facet --mds ${mds}_svc $* - + do_lmc --add mtpt --node ${facet}_facet --clientoptions async --mds ${mds}_svc $* } +config_commit() { + do_lmc --commit +} ####### # General functions @@ -354,16 +510,107 @@ absolute_path() { } ################################## +# OBD_FAIL funcs + +drop_request() { +# OBD_FAIL_MDS_ALL_REQUEST_NET + RC=0 + do_facet mds "echo 0x123 > /proc/sys/lustre/fail_loc" + do_facet client "$1" || RC=$? + do_facet mds "echo 0 > /proc/sys/lustre/fail_loc" + return $RC +} + +drop_reply() { +# OBD_FAIL_MDS_ALL_REPLY_NET + RC=0 + do_facet mds "echo 0x122 > /proc/sys/lustre/fail_loc" + do_facet client "$@" || RC=$? + do_facet mds "echo 0 > /proc/sys/lustre/fail_loc" + return $RC +} + +drop_reint_reply() { +# OBD_FAIL_MDS_REINT_NET_REP + RC=0 + do_facet mds "echo 0x119 > /proc/sys/lustre/fail_loc" + do_facet client "$@" || RC=$? + do_facet mds "echo 0 > /proc/sys/lustre/fail_loc" + return $RC +} + +pause_bulk() { +#define OBD_FAIL_OST_BRW_PAUSE_BULK 0x214 + RC=0 + do_facet ost "echo 0x214 > /proc/sys/lustre/fail_loc" + do_facet client "$1" || RC=$? + do_facet client "sync" + do_facet ost "echo 0 > /proc/sys/lustre/fail_loc" + return $RC +} + +drop_ldlm_cancel() { +#define OBD_FAIL_LDLM_CANCEL 0x304 + RC=0 + do_facet client "echo 0x304 > /proc/sys/lustre/fail_loc" + do_facet client "$@" || RC=$? + do_facet client "echo 0 > /proc/sys/lustre/fail_loc" + return $RC +} + +drop_bl_callback() { +#define OBD_FAIL_LDLM_BL_CALLBACK 0x305 + RC=0 + do_facet client "echo 0x305 > /proc/sys/lustre/fail_loc" + do_facet client "$@" || RC=$? + do_facet client "echo 0 > /proc/sys/lustre/fail_loc" + return $RC +} + +clear_failloc() { + facet=$1 + pause=$2 + sleep $pause + echo "clearing fail_loc on $facet" + do_facet $facet "sysctl -w lustre.fail_loc=0" +} + +cancel_lru_locks() { + $LCTL mark cancel_lru_locks + for d in /proc/fs/lustre/ldlm/namespaces/$1*; do + if [ -f $d/lru_size ]; then + echo clear > $d/lru_size + grep [0-9] $d/lock_unused_count + fi + done +} + + +pgcache_empty() { + for a in /proc/fs/lustre/llite/*/dump_page_cache; do + if [ `wc -l $a | awk '{print $1}'` -gt 1 ]; then + echo there is still data in page cache $a ? + cat $a; + return 1; + fi + done + return 0 +} + +################################## # Test interface error() { echo "${TESTSUITE}: **** FAIL:" $@ + log "FAIL: $@" exit 1 } build_test_filter() { + [ "$ONLY" ] && log "only running $ONLY" for O in $ONLY; do eval ONLY_${O}=true done + [ "$EXCEPT$ALWAYS_EXCEPT" ] && log "skipping $EXCEPT $ALWAYS_EXCEPT" for E in $EXCEPT $ALWAYS_EXCEPT; do eval EXCEPT_${E}=true done @@ -417,14 +664,31 @@ equals_msg() { printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS } +log() { + echo "$*" + lctl mark "$*" 2> /dev/null || true +} + +pass() { + echo PASS $@ +} + run_one() { testnum=$1 message=$2 - tfile=f$base - tdir=d$base + tfile=f${testnum} + tdir=d${base} # Pretty tests run faster. equals_msg $testnum: $message + BEFORE=`date +%s` + log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)" test_${testnum} || error "test_$testnum failed with $?" + pass "($((`date +%s` - $BEFORE))s)" +} + +canonical_path() { + (cd `dirname $1`; echo $PWD/`basename $1`) } +