X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=aeed1dad3aaafac356461ee90f4528a7560d6066;hp=b270e91087735efd3138d99774f59246e792d219;hb=104195e02d53c5e969de776c41e08e3ac4cf9a0f;hpb=13fa1003e4f86b4d814d6259020618684784cbe7 diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index b270e91..aeed1da 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -18,17 +18,9 @@ export IDENTITY_UPCALL=default #export PDSH="pdsh -S -Rssh -w" -# eg, assert_env LUSTRE MDSNODES OSTNODES CLIENTS -assert_env() { - local failed="" - for name in $@; do - if [ -z "${!name}" ]; then - echo "$0: $name must be set" - failed=1 - fi - done - [ $failed ] && exit 1 || true -} +# function used by scripts run on remote nodes +LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} +. $LUSTRE/tests/functions.sh assert_DIR () { local failed="" @@ -108,7 +100,7 @@ init_test_env() { fi export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mdsrate"} [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null) - if ! echo $PATH | grep -q $LUSTRE/test/racer; then + if ! echo $PATH | grep -q $LUSTRE/tests/racer; then export PATH=$PATH:$LUSTRE/tests/racer fi export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} @@ -128,13 +120,13 @@ init_test_env() { export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre) export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " - export FSYTPE=${FSTYPE:-"ldiskfs"} + export FSTYPE=${FSTYPE:-"ldiskfs"} export NAME=${NAME:-local} export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \ export LGSSD=$(which lgssd) export LSVCGSSD=${LSVCGSSD:-"$LUSTRE/utils/gss/lsvcgssd"} - [ ! -f "$LSVCGSSD" ] && export LSVCGSSD=$(which lsvcgssd) + [ ! -f "$LSVCGSSD" ] && export LSVCGSSD=$(which lsvcgssd 2> /dev/null) export KRB5DIR=${KRB5DIR:-"/usr/kerberos"} export DIR2 export SAVE_PWD=${SAVE_PWD:-$LUSTRE/tests} @@ -225,8 +217,8 @@ load_modules() { echo Loading modules from $LUSTRE load_module ../libcfs/libcfs/libcfs - [ "$PTLDEBUG" ] && lctl set_param debug=$PTLDEBUG - [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug=${SUBSYSTEM# } + [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" + [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" local MODPROBECONF= [ -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf [ ! "$MODPROBECONF" -a -d /etc/modprobe.d ] && MODPROBECONF=/etc/modprobe.d/Lustre @@ -252,6 +244,7 @@ load_modules() { load_module mgc/mgc if [ -z "$CLIENTONLY" ] && [ -z "$CLIENTMODSONLY" ]; then grep -q crc16 /proc/kallsyms || { modprobe crc16 2>/dev/null || true; } + grep -q jbd /proc/kallsyms || { modprobe jbd 2>/dev/null || true; } [ "$FSTYPE" = "ldiskfs" ] && load_module ../ldiskfs/ldiskfs/ldiskfs load_module mgs/mgs load_module mds/mds @@ -427,6 +420,10 @@ stop_gss_daemons() { init_gss() { if $GSS; then start_gss_daemons + + if [ -n "$LGSS_KEYRING_DEBUG" ]; then + echo $LGSS_KEYRING_DEBUG > /proc/fs/lustre/sptlrpc/gss/lgss_keyring/debug_level + fi fi } @@ -455,7 +452,7 @@ ostdevlabel() { mount_facet() { local facet=$1 shift - local dev=${facet}_dev + local dev=$(facet_active $facet)_dev local opt=${facet}_opt echo "Starting ${facet}: ${!opt} $@ ${!dev} ${MOUNT%/*}/${facet}" do_facet ${facet} mount -t lustre ${!opt} $@ ${!dev} ${MOUNT%/*}/${facet} @@ -464,8 +461,8 @@ mount_facet() { echo "mount -t lustre $@ ${!dev} ${MOUNT%/*}/${facet}" echo "Start of ${!dev} on ${facet} failed ${RC}" else - do_facet ${facet} "lctl set_param debug=$PTLDEBUG; \ - lctl set_param subsystem_debug=${SUBSYSTEM# }; \ + do_facet ${facet} "lctl set_param debug=\\\"$PTLDEBUG\\\"; \ + lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\"; \ lctl set_param debug_mb=${DEBUG_SIZE}; \ sync" @@ -485,6 +482,14 @@ start() { shift eval export ${facet}_dev=${device} eval export ${facet}_opt=\"$@\" + + local varname=${facet}failover_dev + if [ -n "${!varname}" ] ; then + eval export ${facet}failover_dev=${!varname} + else + eval export ${facet}failover_dev=$device + fi + do_facet ${facet} mkdir -p ${MOUNT%/*}/${facet} mount_facet ${facet} RC=$? @@ -509,6 +514,83 @@ stop() { wait_exit_ST ${facet} } +# save quota version (both administrative and operational quotas) +quota_save_version() { + local fsname=${2:-$FSNAME} + do_facet mgs "lctl conf_param ${fsname}-MDT*.mdd.quota_type=$1" + local varsvc + local osts=$(get_facets OST) + for ost in ${osts//,/ }; do + varsvc=${ost}_svc + do_facet mgs "lctl conf_param ${!varsvc}.ost.quota_type=$1" + done +} + +# client could mount several lustre +quota_type () { + local fsname=${1:-$FSNAME} + local rc=0 + do_facet mgs lctl get_param mdd.${fsname}-MDT*.quota_type || rc=$? + do_nodes $(comma_list $(osts_nodes)) \ + lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$? + return $rc +} + +restore_quota_type () { + local mntpt=${1:-$MOUNT} + local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2) + if [ ! "$old_QUOTA_TYPE" ] || [ "$quota_type" = "$old_QUOTA_TYPE" ]; then + return + fi + $LFS quotaoff $mntpt + quota_save_version $old_QUOTA_TYPE + $LFS quotacheck -ug $mntpt +} + +setup_quota(){ + local mntpt=$1 + + # We need: + # 1. run quotacheck only if quota is off + # 2. save the original quota_type params, restore them after testing + + # Suppose that quota type the same on mds and ost + local quota_type=$(quota_type | grep MDT | cut -d "=" -f2) + [ ${PIPESTATUS[0]} -eq 0 ] || error "quota_type failed!" + if [ "$quota_type" != "$QUOTA_TYPE" ]; then + export old_QUOTA_TYPE=$quota_type + quota_save_version $QUOTA_TYPE + $LFS quotacheck -ug $mntpt + fi + + local quota_usrs=$QUOTA_USERS + + # get_filesystem_size + local disksz=$(lfs df $mntpt | grep "filesystem summary:" | awk '{print $3}') + local blk_soft=$((disksz + 1024)) + local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over + + local Inodes=$(lfs df -i $mntpt | grep "filesystem summary:" | awk '{print $3}') + local i_soft=$Inodes + local i_hard=$((i_soft + i_soft / 20)) + + echo "Total disk size: $disksz block-softlimit: $blk_soft block-hardlimit: + $blk_hard inode-softlimit: $i_soft inode-hardlimit: $i_hard" + + local cmd + for usr in $quota_usrs; do + echo "Setting up quota on $client:$mntpt for $usr..." + for type in u g; do + cmd="$LFS setquota -$type $usr -b $blk_soft -B $blk_hard -i $i_soft -I $i_hard $mntpt" + echo "+ $cmd" + eval $cmd || error "$cmd FAILED!" + done + # display the quota status + echo "Quota settings for $usr : " + $LFS quota -v -u $usr $mntpt || true + done +} + zconf_mount() { local OPTIONS local client=$1 @@ -527,8 +609,8 @@ zconf_mount() { do_node $client mkdir -p $mnt do_node $client mount -t lustre $OPTIONS $device $mnt || return 1 - do_node $client "lctl set_param debug=$PTLDEBUG; - lctl set_param subsystem_debug=${SUBSYSTEM# }; + do_node $client "lctl set_param debug=\\\"$PTLDEBUG\\\"; + lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\"; lctl set_param debug_mb=${DEBUG_SIZE}" return 0 @@ -561,6 +643,62 @@ zconf_umount() { fi } +# nodes is comma list +sanity_mount_check_nodes () { + local nodes=$1 + shift + local mnts="$@" + local mnt + + # FIXME: assume that all cluster nodes run the same os + [ "$(uname)" = Linux ] || return 0 + + local rc=0 + for mnt in $mnts ; do + do_nodes $nodes "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts); +mpts=\\\$(mount | grep -w -c $mnt); +if [ \\\$running -ne \\\$mpts ]; then + echo \\\$(hostname) env are INSANE!; + exit 1; +fi" + [ $? -eq 0 ] || rc=1 + done + return $rc +} + +sanity_mount_check_servers () { + echo Checking servers environments + + # FIXME: modify get_facets to display all facets wo params + local facets="$(get_facets OST),$(get_facets MDS)" + local node + local mnt + local facet + for facet in ${facets//,/ }; do + node=$(facet_host ${facet}) + mnt=${MOUNT%/*}/${facet} + sanity_mount_check_nodes $node $mnt || + { error "server $node environments are insane!"; return 1; } + done +} + +sanity_mount_check_clients () { + local clients=${1:-$CLIENTS} + local mntpt=${2:-$MOUNT} + local mntpt2=${3:-$MOUNT2} + + [ -z $clients ] && clients=$(hostname) + echo Checking clients $clients environments + + sanity_mount_check_nodes $clients $mntpt $mntpt2 || + error "clients environments are insane!" +} + +sanity_mount_check () { + sanity_mount_check_servers || return 1 + sanity_mount_check_clients || return 2 +} + # mount clients if not mouted zconf_mount_clients() { local OPTIONS @@ -579,13 +717,22 @@ zconf_mount_clients() { fi echo "Starting client $clients: $OPTIONS $device $mnt" - do_nodes $clients "mount | grep $mnt || { mkdir -p $mnt && mount -t lustre $OPTIONS $device $mnt || false; }" + + do_nodes $clients "set -x; +running=\\\$(mount | grep -c $mnt' '); +rc=0; +if [ \\\$running -eq 0 ] ; then + mkdir -p $mnt; + mount -t lustre $OPTIONS $device $mnt; + rc=$?; +fi; +exit $rc" echo "Started clients $clients: " - do_nodes $clients "mount | grep $mnt" + do_nodes $clients "mount | grep -w $mnt" - do_nodes $clients "lctl set_param debug=$PTLDEBUG; - lctl set_param subsystem_debug=${SUBSYSTEM# }; + do_nodes $clients "lctl set_param debug=\\\"$PTLDEBUG\\\"; + lctl set_param subsystem_debug=\\\"${SUBSYSTEM# }\\\"; lctl set_param debug_mb=${DEBUG_SIZE};" return 0 @@ -599,29 +746,54 @@ zconf_umount_clients() { [ "$3" ] && force=-f echo "Stopping clients: $clients $mnt (opts:$force)" - do_nodes $clients "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts) + do_nodes $clients "set -x; running=\\\$(grep -c $mnt' ' /proc/mounts); if [ \\\$running -ne 0 ] ; then -echo Stopping client \\\$(hostname) client $mnt opts:$force -lsof -t $mnt || need_kill=no +echo Stopping client \\\$(hostname) client $mnt opts:$force; +lsof -t $mnt || need_kill=no; if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then pids=\\\$(lsof -t $mnt | sort -u); - if [ -n \\\$pids ]; then - kill -9 \\\$pids + if [ -n \\\"\\\$pids\\\" ]; then + kill -9 \\\$pids; fi -fi -busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy") +fi; +busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy"); if [ \\\$busy -ne 0 ] ; then - echo "$mnt is still busy, wait one second" && sleep 1 - umount $force $mnt + echo "$mnt is still busy, wait one second" && sleep 1; + umount $force $mnt; fi fi" } +shudown_node_hard () { + local host=$1 + local attempts=3 + + for i in $(seq $attempts) ; do + $POWER_DOWN $host + sleep 1 + ping -w 3 -c 1 $host > /dev/null 2>&1 || return 0 + echo "waiting for $host to fail attempts=$attempts" + [ $i -lt $attempts ] || \ + { echo "$host still pingable after power down! attempts=$attempts" && return 1; } + done +} + +shutdown_client() { + local client=$1 + local mnt=${2:-$MOUNT} + local attempts=3 + + if [ "$FAILURE_MODE" = HARD ]; then + shudown_node_hard $client + else + zconf_umount_clients $client $mnt -f + fi +} + shutdown_facet() { local facet=$1 if [ "$FAILURE_MODE" = HARD ]; then - $POWER_DOWN `facet_active_host $facet` - sleep 2 + shudown_node_hard $(facet_active_host $facet) elif [ "$FAILURE_MODE" = SOFT ]; then stop $facet fi @@ -655,39 +827,43 @@ check_progs_installed () { eval \\\$status" } -start_client_load() { - local list=(${1//,/ }) - local nodenum=$2 +client_var_name() { + echo __$(echo $1 | tr '-' 'X') +} - local numloads=${#CLIENT_LOADS[@]} - local testnum=$((nodenum % numloads)) +start_client_load() { + local client=$1 + local load=$2 + local var=$(client_var_name $client)_load + eval export ${var}=$load - do_node ${list[nodenum]} "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ + do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ BREAK_ON_ERROR=$BREAK_ON_ERROR \ END_RUN_FILE=$END_RUN_FILE \ LOAD_PID_FILE=$LOAD_PID_FILE \ TESTSUITELOG=$TESTSUITELOG \ - run_${CLIENT_LOADS[testnum]}.sh" & + run_${load}.sh" & CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $!" - log "Started client load: ${CLIENT_LOADS[testnum]} on ${list[nodenum]}" + log "Started client load: ${load} on $client" - eval export ${list[nodenum]}_load=${CLIENT_LOADS[testnum]} return 0 } start_client_loads () { - local clients=(${1//,/ }) + local -a clients=(${1//,/ }) + local numloads=${#CLIENT_LOADS[@]} + local testnum - for ((num=0; num < ${#clients[@]}; num++ )); do - start_client_load $1 $num + for ((nodenum=0; nodenum < ${#clients[@]}; nodenum++ )); do + testnum=$((nodenum % numloads)) + start_client_load ${clients[nodenum]} ${CLIENT_LOADS[testnum]} done } # only for remote client check_client_load () { local client=$1 - local var=${client}_load - + local var=$(client_var_name $client)_load local TESTLOAD=run_${!var}.sh ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1 @@ -720,13 +896,39 @@ check_client_loads () { for client in $clients; do check_client_load $client - rc=$? + rc=${PIPESTATUS[0]} if [ "$rc" != 0 ]; then log "Client load failed on node $client, rc=$rc" return $rc fi done } + +restart_client_loads () { + local clients=${1//,/ } + local expectedfail=${2:-""} + local client= + local rc=0 + + for client in $clients; do + check_client_load $client + rc=${PIPESTATUS[0]} + if [ "$rc" != 0 -a "$expectedfail" ]; then + start_client_load $client + echo "Restarted client load: on $client. Checking ..." + check_client_load $client + rc=${PIPESTATUS[0]} + if [ "$rc" != 0 ]; then + log "Client load failed to restart on node $client, rc=$rc" + # failure one client load means test fail + # we do not need to check other + return $rc + fi + else + return $rc + fi + done +} # End recovery-scale functions # verify that lustre actually cleaned up properly @@ -753,6 +955,35 @@ cleanup_check() { return 0 } +wait_update () { + local node=$1 + local TEST=$2 + local FINAL=$3 + local MAX=${4:-90} + + local RESULT + local WAIT=0 + local sleep=5 + while [ true ]; do + RESULT=$(do_node $node "$TEST") + if [ "$RESULT" == "$FINAL" ]; then + echo "Updated after $WAIT sec: wanted '$FINAL' got '$RESULT'" + return 0 + fi + [ $WAIT -ge $MAX ] && break + echo "Waiting $((MAX - WAIT)) secs for update" + WAIT=$((WAIT + sleep)) + sleep $sleep + done + echo "Update not seen after $MAX sec: wanted '$FINAL' got '$RESULT'" + return 3 +} + +wait_update_facet () { + local facet=$1 + wait_update $(facet_active_host $facet) "$@" +} + wait_delete_completed () { local TOTALPREV=`lctl get_param -n osc.*.kbytesavail | \ awk 'BEGIN{total=0}; {total+=$1}; END{print total}'` @@ -772,32 +1003,39 @@ wait_delete_completed () { } wait_for_host() { - HOST=$1 - check_network "$HOST" 900 - while ! do_node $HOST "ls -d $LUSTRE " > /dev/null; do sleep 5; done + local host=$1 + check_network "$host" 900 + while ! do_node $host "ls -d $LUSTRE " > /dev/null; do sleep 5; done } wait_for() { - facet=$1 - HOST=`facet_active_host $facet` - wait_for_host $HOST + local facet=$1 + local host=`facet_active_host $facet` + wait_for_host $host } -wait_mds_recovery_done () { - local timeout=`do_facet $SINGLEMDS lctl get_param -n timeout` -#define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2) -# as we are in process of changing obd_timeout in different ways -# let's set MAX longer than that - MAX=$(( timeout * 4 )) - WAIT=0 +wait_recovery_complete () { + local facet=$1 + + # Use default policy if $2 is not passed by caller. + #define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2) + # as we are in process of changing obd_timeout in different ways + # let's set MAX longer than that + local MAX=${2:-$(( TIMEOUT * 4 ))} + + local var_svc=${facet}_svc + local procfile="*.${!var_svc}.recovery_status" + local WAIT=0 + local STATUS= + while [ $WAIT -lt $MAX ]; do - STATUS=`do_facet $SINGLEMDS "lctl get_param -n mdt.*-MDT0000.recovery_status | grep status"` - echo $STATUS | grep COMPLETE && return 0 + STATUS=$(do_facet $facet lctl get_param -n $procfile | grep status) + [[ $STATUS = "status: COMPLETE" ]] && return 0 sleep 5 WAIT=$((WAIT + 5)) - echo "Waiting $(($MAX - $WAIT)) secs for MDS recovery done" + echo "Waiting $((MAX - WAIT)) secs for $facet recovery done. $STATUS" done - echo "MDS recovery not done in $MAX sec" + echo "$facet recovery not done in $MAX sec. $STATUS" return 1 } @@ -841,6 +1079,7 @@ wait_remote_prog () { local pids=$(ps uax | grep "$PDSH.*$prog.*$MOUNT" | grep -v grep | awk '{print $2}') [ -z "$pids" ] && return 0 echo "$PDSH processes still exists after $WAIT seconds. Still running: $pids" + # FIXME: not portable for pid in $pids; do cat /proc/${pid}/status || true cat /proc/${pid}/wchan || true @@ -876,8 +1115,8 @@ client_reconnect() { } facet_failover() { - facet=$1 - sleep_time=$2 + local facet=$1 + local sleep_time=$2 echo "Failing $facet on node `facet_active_host $facet`" shutdown_facet $facet [ -n "$sleep_time" ] && sleep $sleep_time @@ -886,7 +1125,7 @@ facet_failover() { DFPID=$! echo "df pid is $DFPID" change_active $facet - TO=`facet_active_host $facet` + local TO=`facet_active_host $facet` echo "Failover $facet to $TO" wait_for $facet mount_facet $facet || error "Restart of $facet failed" @@ -901,8 +1140,8 @@ replay_barrier() { do_facet $facet sync df $MOUNT local svc=${facet}_svc - do_facet $facet $LCTL --device %${!svc} readonly do_facet $facet $LCTL --device %${!svc} notransno + do_facet $facet $LCTL --device %${!svc} readonly do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}" $LCTL mark "local REPLAY BARRIER on ${!svc}" } @@ -912,8 +1151,8 @@ replay_barrier_nodf() { do_facet $facet sync local svc=${facet}_svc echo Replay barrier on ${!svc} - do_facet $facet $LCTL --device %${!svc} readonly do_facet $facet $LCTL --device %${!svc} notransno + do_facet $facet $LCTL --device %${!svc} readonly do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}" $LCTL mark "local REPLAY BARRIER on ${!svc}" } @@ -922,8 +1161,8 @@ replay_barrier_nosync() { local facet=$1 echo running=${running} local svc=${facet}_svc echo Replay barrier on ${!svc} - do_facet $facet $LCTL --device %${!svc} readonly do_facet $facet $LCTL --device %${!svc} notransno + do_facet $facet $LCTL --device %${!svc} readonly do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}" $LCTL mark "local REPLAY BARRIER on ${!svc}" } @@ -1056,17 +1295,17 @@ facet_active_host() { change_active() { local facet=$1 - failover=${facet}failover + local failover=${facet}failover host=`facet_host $failover` [ -z "$host" ] && return - curactive=`facet_active $facet` + local curactive=`facet_active $facet` if [ -z "${curactive}" -o "$curactive" == "$failover" ] ; then eval export ${facet}active=$facet else eval export ${facet}active=$failover fi # save the active host for this facet - activevar=${facet}active + local activevar=${facet}active echo "$activevar=${!activevar}" > $TMP/$activevar } @@ -1128,9 +1367,9 @@ do_nodes() { } do_facet() { - facet=$1 + local facet=$1 shift - HOST=`facet_active_host $facet` + local HOST=`facet_active_host $facet` [ -z $HOST ] && echo No host defined for facet ${facet} && exit 1 do_node $HOST "$@" } @@ -1171,14 +1410,11 @@ stopall() { fail mds1 fi - # assume client mount is local - grep " $MOUNT " /proc/mounts && zconf_umount $HOSTNAME $MOUNT $* - grep " $MOUNT2 " /proc/mounts && zconf_umount $HOSTNAME $MOUNT2 $* + local clients=$CLIENTS + [ -z $clients ] && clients=$(hostname) - if [ -n "$CLIENTS" ]; then - zconf_umount_clients $CLIENTS $MOUNT "$*" || true - [ -n "$MOUNT2" ] && zconf_umount_clients $CLIENTS $MOUNT2 "$*" || true - fi + zconf_umount_clients $clients $MOUNT "$*" || true + [ -n "$MOUNT2" ] && zconf_umount_clients $clients $MOUNT2 "$*" || true [ "$CLIENTONLY" ] && return # The add fn does rm ${facet}active file, this would be enough @@ -1292,16 +1528,6 @@ remount_client() zconf_mount `hostname` $1 || error "mount failed" } -set_obd_timeout() { - local facet=$1 - local timeout=$2 - - do_facet $facet lsmod | grep -q obdclass || \ - do_facet $facet "modprobe obdclass" - - do_facet $facet "lctl set_param timeout=$timeout" -} - writeconf_facet () { local facet=$1 local dev=$2 @@ -1322,6 +1548,9 @@ writeconf_all () { } setupall() { + sanity_mount_check || + error "environments are insane!" + load_modules init_gss if [ -z "$CLIENTONLY" ]; then @@ -1330,7 +1559,6 @@ setupall() { writeconf_all for num in `seq $MDSCOUNT`; do DEVNAME=$(mdsdevname $num) - set_obd_timeout mds$num $TIMEOUT start mds$num $DEVNAME $MDS_MOUNT_OPTS # We started mds, now we should set failover variables properly. @@ -1346,7 +1574,6 @@ setupall() { done for num in `seq $OSTCOUNT`; do DEVNAME=$(ostdevname $num) - set_obd_timeout ost$num $TIMEOUT start ost$num $DEVNAME $OST_MOUNT_OPTS # We started ost$num, now we should set ost${num}failover variable properly. @@ -1371,7 +1598,7 @@ setupall() { [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2 fi - init_versions_vars + init_param_vars # by remounting mdt before ost, initial connect from mdt to ost might # timeout because ost is not ready yet. wait some time to its fully @@ -1409,15 +1636,27 @@ init_facet_vars () { if [ -z "${!varname}" ]; then eval $varname=$(facet_host $facet) fi + + # ${facet}failover_dev is set in cfg file + varname=${facet}failover_dev + if [ -n "${!varname}" ] ; then + eval export ${facet}failover_dev=${!varname} + else + eval export ${facet}failover_dev=$device + fi } init_facets_vars () { local DEVNAME - for num in `seq $MDSCOUNT`; do - DEVNAME=`mdsdevname $num` - init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS - done + if ! remote_mds_nodsh; then + for num in `seq $MDSCOUNT`; do + DEVNAME=`mdsdevname $num` + init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS + done + fi + + remote_ost_nodsh && return for num in `seq $OSTCOUNT`; do DEVNAME=`ostdevname $num` @@ -1425,10 +1664,21 @@ init_facets_vars () { done } -init_versions_vars () { - export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2) - export OSTVER=$(do_facet ost1 "lctl get_param version" | cut -d. -f1,2) - export CLIVER=$(lctl get_param version | cut -d. -f 1,2) +init_param_vars () { + if ! remote_ost_nodsh && ! remote_mds_nodsh; then + export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2) + export OSTVER=$(do_facet ost1 "lctl get_param version" | cut -d. -f1,2) + export CLIVER=$(lctl get_param version | cut -d. -f 1,2) + fi + + remote_mds_nodsh || + TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") + + log "Using TIMEOUT=$TIMEOUT" + + if [ "$ENABLE_QUOTA" ]; then + setup_quota $MOUNT || return 2 + fi } check_config () { @@ -1447,6 +1697,18 @@ check_config () { error "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE Please use correct config or set mds_HOST correctly!" fi + + sanity_mount_check || + error "environments are insane!" +} + +check_timeout () { + local mdstimeout=$(do_facet $SINGLEMDS "lctl get_param -n timeout") + local cltimeout=$(lctl get_param -n timeout) + if [ $mdstimeout -ne $TIMEOUT ] || [ $mdstimeout -ne $cltimeout ]; then + error "timeouts are wrong! mds: $mdstimeout, client: $cltimeout, TIMEOUT=$TIMEOUT" + return 1 + fi } check_and_setup_lustre() { @@ -1460,7 +1722,7 @@ check_and_setup_lustre() { else check_config $MOUNT init_facets_vars - init_versions_vars + init_param_vars fi if [ "$ONLY" == "setup" ]; then exit 0 @@ -1481,6 +1743,7 @@ cleanup_and_setup_lustre() { check_and_cleanup_lustre() { if [ "`mount | grep $MOUNT`" ]; then [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* + [ "$ENABLE_QUOTA" ] && restore_quota_type || true fi if [ "$I_MOUNTED" = "yes" ]; then cleanupall -f || error "cleanup failed" @@ -1527,20 +1790,45 @@ comma_list() { echo "$*" | tr -s " " "\n" | sort -b -u | tr "\n" " " | sed 's/ \([^$]\)/,\1/g' } -# list is comma separated list -exclude_item_from_list () { +# list, excluded are the comma separated lists +exclude_items_from_list () { local list=$1 local excluded=$2 + local item list=${list//,/ } - list=$(echo " $list " | sed -re "s/\s+$excluded\s+/ /g") + for item in ${excluded//,/ }; do + list=$(echo " $list " | sed -re "s/\s+$item\s+/ /g") + done echo $(comma_list $list) } +# list, expand are the comma separated lists +expand_list () { + local list=${1//,/ } + local expand=${2//,/ } + local expanded= + + expanded=$(for i in $list $expand; do echo $i; done | sort -u) + echo $(comma_list $expanded) +} + absolute_path() { (cd `dirname $1`; echo $PWD/`basename $1`) } +get_facets () { + local name=$(echo $1 | tr "[:upper:]" "[:lower:]") + local type=$(echo $1 | tr "[:lower:]" "[:upper:]") + + local list="" + local count=${type}COUNT + for ((i=1; i<=${!count}; i++)) do + list="$list ${name}$i" + done + echo $(comma_list $list) +} + ################################## # Adaptive Timeouts funcs @@ -1742,7 +2030,7 @@ error_noexit() { ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s) echo "Dumping lctl log to $ERRLOG" # We need to dump the logs on all nodes - local NODES=$(nodes_list) + local NODES=${NODES:-$(nodes_list)} for NODE in $NODES; do do_node $NODE $LCTL dk $ERRLOG done @@ -1771,9 +2059,10 @@ error_ignore() { } skip () { - log " SKIP: ${TESTSUITE} ${TESTNAME} $@" - [ "$TESTSUITELOG" ] && \ - echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true + echo + log " SKIP: ${TESTSUITE} ${TESTNAME} $@" + [ "$TESTSUITELOG" ] && \ + echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true } build_test_filter() { @@ -1872,7 +2161,7 @@ log() { lsmod | grep lnet > /dev/null || load_modules local MSG="$*" - # Get rif of ' + # Get rid of ' MSG=${MSG//\'/\\\'} MSG=${MSG//\(/\\\(} MSG=${MSG//\)/\\\)} @@ -1910,9 +2199,11 @@ reset_fail_loc () { local myNODES=$(nodes_list) local NODE + echo -n "Resetting fail_loc on all nodes..." for NODE in $myNODES; do - do_node $NODE "lctl set_param fail_loc=0 2>/dev/null || true" + do_node $NODE "lctl set_param -n fail_loc=0 2>/dev/null || true" done + echo done. } run_one() { @@ -1924,8 +2215,9 @@ run_one() { local SAVE_UMASK=`umask` umask 0022 - BEFORE=`date +%s` - log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)" + local BEFORE=`date +%s` + echo + log "== test $testnum: $message == `date +%H:%M:%S` ($BEFORE)" #check_mds export TESTNAME=test_$testnum TEST_FAILED=false @@ -2042,6 +2334,13 @@ remote_ost_nodsh() remote_ost && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] } +remote_mgs_nodsh() +{ + local MGS + MGS=$(facet_host mgs) + remote_node $MGS && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] +} + mdts_nodes () { local MDSNODES local NODES_sort @@ -2121,9 +2420,9 @@ get_random_entry () { rnodes=${rnodes//,/ } - local nodes=($rnodes) + local -a nodes=($rnodes) local num=${#nodes[@]} - local i=$((RANDOM * num / 65536)) + local i=$((RANDOM * num * 2 / 65536)) echo ${nodes[i]} } @@ -2231,25 +2530,9 @@ multiop_bg_pause() { return 0 } -check_rate() { - local OP=$1 - local TARGET_RATE=$2 - local NUM_CLIENTS=$3 - local LOG=$4 - - local RATE=$(awk '/^Rate: [0-9\.]+ '"${OP}"'s\/sec/ { print $2}' ${LOG}) - - # We need to use bc since the rate is a floating point number - local RES=$(echo "${RATE} < ${TARGET_RATE}" | bc -l ) - if [ "${RES}" = 0 ]; then - echo "Success: ${RATE} ${OP}s/sec met target rate" \ - "${TARGET_RATE} ${OP}s/sec for ${NUM_CLIENTS} client(s)." - return 0 - else - echo "Failure: ${RATE} ${OP}s/sec did not meet target rate" \ - "${TARGET_RATE} ${OP}s/sec for ${NUM_CLIENTS} client(s)." - return 1 - fi +inodes_available () { + local IFree=$($LFS df -i $MOUNT | grep ^$FSNAME | awk '{print $4}' | sort -un | head -1) || return 1 + echo $IFree } # reset llite stat counters @@ -2281,6 +2564,7 @@ calc_sum () { } calc_osc_kbytes () { + df $MOUNT > /dev/null $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum } @@ -2301,17 +2585,21 @@ restore_lustre_params() { done } -check_catastrophe () { +check_catastrophe() { local rnodes=${1:-$(comma_list $(remote_nodes_list))} + local C=$CATASTROPHE + [ -f $C ] && [ $(cat $C) -ne 0 ] && return 1 - [ -f $CATASTROPHE ] && [ $(cat $CATASTROPHE) -ne 0 ] && return 1 if [ $rnodes ]; then - do_nodes $rnodes "set -x; [ -f $CATASTROPHE ] && { [ \`cat $CATASTROPHE\` -eq 0 ] || false; } || true" + do_nodes $rnodes "rc=\\\$([ -f $C ] && echo \\\$(< $C) || echo 0); +if [ \\\$rc -ne 0 ]; then echo \\\$(hostname): \\\$rc; fi +exit \\\$rc;" fi } # $1 node # $2 file +# $3 $RUNAS get_stripe_info() { local tmp_file @@ -2320,7 +2608,7 @@ get_stripe_info() { stripe_index=0 tmp_file=$(mktemp) - do_facet $1 lfs getstripe -v $2 > $tmp_file + do_facet $1 $3 lfs getstripe -v $2 > $tmp_file stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file` stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file` @@ -2331,9 +2619,10 @@ get_stripe_info() { # CMD: determine mds index where directory inode presents get_mds_dir () { local dir=$1 - local file=$dir/$tfile + local file=$dir/f0.get_mds_dir_tmpfile rm -f $file + sleep 1 local iused=$(lfs df -i $dir | grep MDT | awk '{print $3}') local oldused=($iused) @@ -2346,7 +2635,7 @@ get_mds_dir () { for ((i=0; i<${#newused[@]}; i++)); do if [ ${oldused[$i]} -lt ${newused[$i]} ]; then echo $(( i + 1 )) - rm -f $dir/$tfile + rm -f $file return 0 fi done @@ -2356,6 +2645,8 @@ get_mds_dir () { mpi_run () { local mpirun="$MPIRUN $MPIRUN_OPTIONS" local command="$mpirun $@" + local mpilog=$TMP/mpi.log + local rc if [ "$MPI_USER" != root -a $mpirun ]; then echo "+ chmod 0777 $MOUNT" @@ -2365,6 +2656,106 @@ mpi_run () { ls -ald $MOUNT echo "+ $command" - eval $command + eval $command 2>&1 > $mpilog || true + + rc=${PIPESTATUS[0]} + if [ $rc -eq 0 ] && grep -q "p4_error: : [^0]" $mpilog ; then + rc=1 + fi + cat $mpilog + return $rc } +mdsrate_cleanup () { + mpi_run -np $1 -machinefile $2 ${MDSRATE} --unlink --nfiles $3 --dir $4 --filefmt $5 $6 +} + +delayed_recovery_enabled () { + local var=${SINGLEMDS}_svc + do_facet $SINGLEMDS lctl get_param -n mdd.${!var}.stale_export_age > /dev/null 2>&1 +} + +######################## +convert_facet2name() { + case "$1" in + "ost" ) echo "OST0000" ;; + "ost1") echo "OST0000" ;; + "ost2") echo "OST0001" ;; + "ost3") echo "OST0002" ;; + "ost4") echo "OST0003" ;; + "ost5") echo "OST0004" ;; + *) error "unknown facet!" ;; + esac +} + +get_clientosc_proc_path() { + local ost=$1 + + echo "{$1}-osc-*" +} + +get_lustre_version () { + local node=${1:-"mds"} + do_facet $node $LCTL get_param -n version | awk '/^lustre:/ {print $2}' +} + +get_mds_version_major () { + local version=$(get_lustre_version mds) + echo $version | awk -F. '{print $1}' +} + +get_mds_version_minor () { + local version=$(get_lustre_version mds) + echo $version | awk -F. '{print $2}' +} + +get_mdtosc_proc_path() { + local ost=$1 + local major=$(get_mds_version_major) + local minor=$(get_mds_version_minor) + if [ $major -le 1 -a $minor -le 8 ] ; then + echo "${ost}-osc" + else + echo "${ost}-osc-MDT0000" + fi +} + +get_osc_import_name() { + local node=$1 + local ost=$2 + local name=$(convert_facet2name $ost) + + if [ "$node" == "mds" ]; then + get_mdtosc_proc_path $name + return 0 + fi + + get_clientosc_proc_path $name + return 0 +} + +wait_osc_import_state() { + local node=$1 + local ost_facet=$2 + local expected=$3 + local ost=$(get_osc_import_name $node $ost_facet) + local CONN_PROC + local CONN_STATE + local i=0 + + CONN_PROC="osc.${FSNAME}-${ost}.ost_server_uuid" + CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC | cut -f2) + while [ "${CONN_STATE}" != "${expected}" ]; do + # for disconn we can check after proc entry is removed + [ "x${CONN_STATE}" == "x" -a "${expected}" == "DISCONN" ] && return 0 + # disconnect rpc should be wait not more obd_timeout + [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \ + error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1 + sleep 1 + CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC | cut -f2) + i=$(($i + 1)) + done + + log "${ost_facet} now in ${CONN_STATE} state" + return 0 +}