X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=sidebyside;f=lustre%2Ftests%2Ftest-framework.sh;h=d3a27c99a21408c0b03e45b0544f493e8ec1bef0;hb=df19e89b6c2c5b355ab2156da99bebf939ee01fd;hp=ad6ad36e03eefed2ee25bdd220d479e5bd2e4c55;hpb=322968acf183ab16d952cd3026f6580957b31259;p=fs%2Flustre-release.git diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index ad6ad36..d3a27c9 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1,5 +1,6 @@ #!/bin/bash -# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: +# -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*- +# vim:shiftwidth=4:softtabstop=4:tabstop=4: trap 'print_summary && touch $TF_FAIL && \ echo "test-framework exiting on error"' ERR @@ -133,10 +134,9 @@ init_test_env() { #[ -d /r ] && export ROOT=${ROOT:-/r} export TMP=${TMP:-$ROOT/tmp} export TESTSUITELOG=${TMP}/${TESTSUITE}.log - if [[ -z $LOGDIRSET ]]; then - export LOGDIR=${LOGDIR:-${TMP}/test_logs/}/$(date +%s) - export LOGDIRSET=true - fi + export LOGDIR=${LOGDIR:-${TMP}/test_logs/$(date +%s)} + export TESTLOG_PREFIX=$LOGDIR/$TESTSUITE + export HOSTNAME=${HOSTNAME:-$(hostname -s)} if ! echo $PATH | grep -q $LUSTRE/utils; then export PATH=$LUSTRE/utils:$PATH @@ -166,10 +166,14 @@ init_test_env() { export PATH=$LUSTRE/tests/mpi:$PATH fi export RSYNC_RSH=${RSYNC_RSH:-rsh} + export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} [ ! -f "$LCTL" ] && export LCTL=$(which lctl) export LFS=${LFS:-"$LUSTRE/utils/lfs"} [ ! -f "$LFS" ] && export LFS=$(which lfs) + SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"} + GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"} + export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"} if [ ! -f "$L_GETIDENTITY" ]; then if `which l_getidentity > /dev/null 2>&1`; then @@ -186,7 +190,11 @@ init_test_env() { [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre) export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod} - [ ! -f "$LUSTRE_RMMOD" ] && export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null) + [ ! -f "$LUSTRE_RMMOD" ] && + export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null) + export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate} + [ ! -f "$LFS_MIGRATE" ] && + export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null) export FSTYPE=${FSTYPE:-"ldiskfs"} export NAME=${NAME:-local} export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} @@ -256,6 +264,16 @@ init_test_env() { rm -f $TMP/*active } +version_code() { + # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3" + eval set -- $(tr "[:punct:]" " " <<< $*) + + echo -n $((($1 << 16) | ($2 << 8) | $3)) +} + +export LINUX_VERSION=$(uname -r | sed -e "s/[-.]/ /3" -e "s/ .*//") +export LINUX_VERSION_CODE=$(version_code ${LINUX_VERSION//\./ }) + case `uname -r` in 2.4.*) EXT=".o"; USE_QUOTA=no; [ ! "$CLIENTONLY" ] && FSTYPE=ext3;; *) EXT=".ko"; USE_QUOTA=yes;; @@ -360,7 +378,6 @@ load_modules_local() { load_module obdclass/obdclass load_module ptlrpc/ptlrpc load_module ptlrpc/gss/ptlrpc_gss - [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS load_module fld/fld load_module fid/fid load_module lmv/lmv @@ -378,6 +395,7 @@ load_modules_local() { { modprobe exportfs 2> /dev/null || true; } load_module ../ldiskfs/ldiskfs/ldiskfs fi + [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS load_module mgs/mgs load_module mds/mds load_module mdd/mdd @@ -386,7 +404,11 @@ load_modules_local() { load_module cmm/cmm load_module osd-ldiskfs/osd_ldiskfs load_module ost/ost - load_module obdfilter/obdfilter + if [ "x$USE_OFD" = "xyes" ]; then + load_module ofd/ofd + else + load_module obdfilter/obdfilter + fi fi @@ -584,7 +606,12 @@ ostdevlabel() { set_debug_size () { local dz=${1:-$DEBUG_SIZE} - local cpus=$(getconf _NPROCESSORS_CONF) + + if [ -f /sys/devices/system/cpu/possible ]; then + local cpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/possible)+1)) + else + local cpus=$(getconf _NPROCESSORS_CONF) + fi # bug 19944, adjust size to be -gt num_possible_cpus() # promise 2MB for every cpu at least @@ -596,14 +623,13 @@ set_debug_size () { set_default_debug () { local debug=${1:-"$PTLDEBUG"} - local subsystem_debug=${2:-"$SUBSYSTEM"} + local subsys=${2:-"$SUBSYSTEM"} local debug_size=${3:-$DEBUG_SIZE} - lctl set_param debug="$debug" - lctl set_param subsystem_debug="${subsystem_debug# }" + [ -n "$debug" ] && lctl set_param debug="$debug" >/dev/null + [ -n "$subsys" ] && lctl set_param subsystem_debug="${subsys# }" >/dev/null - set_debug_size $debug_size - sync + [ -n "$debug_size" ] && set_debug_size $debug_size > /dev/null } set_default_debug_nodes () { @@ -628,12 +654,21 @@ set_default_debug_facet () { # Facet functions mount_facets () { - local facets=${1:-$(get_facets)} - local facet + local facets=${1:-$(get_facets)} + local facet - for facet in ${facets//,/ }; do - mount_facet $facet || error "Restart of $facet failed!" - done + for facet in ${facets//,/ }; do + mount_facet $facet + local RC=$? + [ $RC -eq 0 ] && continue + + if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then + skip "Restart of $facet failed!." && touch $LU482_FAILED + else + error "Restart of $facet failed!" + fi + return $RC + done } mount_facet() { @@ -646,6 +681,8 @@ mount_facet() { echo "Starting ${facet}: ${!opt} $@ ${!dev} $mntpt" do_facet ${facet} "mkdir -p $mntpt; mount -t lustre ${!opt} $@ ${!dev} $mntpt" RC=${PIPESTATUS[0]} + # to allow testing LU-482 error handling in mount_facets() and test_0a() + [ -f $TMP/test-lu482-trigger ] && RC=2 if [ $RC -ne 0 ]; then echo "mount -t lustre $@ ${!dev} $mntpt" echo "Start of ${!dev} on ${facet} failed ${RC}" @@ -848,7 +885,7 @@ sanity_mount_check_nodes () { local rc=0 for mnt in $mnts ; do do_nodes $nodes "running=\\\$(grep -c $mnt' ' /proc/mounts); -mpts=\\\$(mount | grep -w -c $mnt); +mpts=\\\$(mount | grep -c $mnt' '); if [ \\\$running -ne \\\$mpts ]; then echo \\\$(hostname) env are INSANE!; exit 1; @@ -918,7 +955,7 @@ fi; exit \\\$rc" || return ${PIPESTATUS[0]} echo "Started clients $clients: " - do_nodes $clients "mount | grep -w $mnt" + do_nodes $clients "mount | grep $mnt' '" set_default_debug_nodes $clients @@ -936,18 +973,16 @@ zconf_umount_clients() { do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts); if [ \\\$running -ne 0 ] ; then echo Stopping client \\\$(hostname) $mnt opts:$force; -lsof -t $mnt || need_kill=no; +lsof $mnt || need_kill=no; if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then pids=\\\$(lsof -t $mnt | sort -u); if [ -n \\\"\\\$pids\\\" ]; then kill -9 \\\$pids; fi fi; -busy=\\\$(umount $force $mnt 2>&1 | grep -c "busy"); -if [ \\\$busy -ne 0 ] ; then +while umount $force $mnt 2>&1 | grep -q "busy"; do echo "$mnt is still busy, wait one second" && sleep 1; - umount $force $mnt; -fi +done; fi" } @@ -1104,14 +1139,20 @@ start_client_load() { eval export ${var}=$load do_node $client "PATH=$PATH MOUNT=$MOUNT ERRORS_OK=$ERRORS_OK \ - BREAK_ON_ERROR=$BREAK_ON_ERROR \ - END_RUN_FILE=$END_RUN_FILE \ - LOAD_PID_FILE=$LOAD_PID_FILE \ - TESTSUITELOG=$TESTSUITELOG \ - run_${load}.sh" & - CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $!" +BREAK_ON_ERROR=$BREAK_ON_ERROR \ +END_RUN_FILE=$END_RUN_FILE \ +LOAD_PID_FILE=$LOAD_PID_FILE \ +TESTLOG_PREFIX=$TESTLOG_PREFIX \ +TESTNAME=$TESTNAME \ +DBENCH_LIB=$DBENCH_LIB \ +DBENCH_SRC=$DBENCH_SRC \ +run_${load}.sh" & + local ppid=$! log "Started client load: ${load} on $client" + # get the children process IDs + local pids=$(ps --ppid $ppid -o pid= | xargs) + CLIENT_LOAD_PIDS="$CLIENT_LOAD_PIDS $ppid $pids" return 0 } @@ -1128,14 +1169,14 @@ start_client_loads () { sleep 2 } -# only for remote client +# only for remote client check_client_load () { local client=$1 local var=$(node_var_name $client)_load local TESTLOAD=run_${!var}.sh ps auxww | grep -v grep | grep $client | grep -q "$TESTLOAD" || return 1 - + # bug 18914: try to connect several times not only when # check ps, but while check_catastrophe also local tries=3 @@ -1213,7 +1254,7 @@ restart_client_loads () { if [ "$rc" != 0 ]; then log "Client load failed to restart on node $client, rc=$rc" # failure one client load means test fail - # we do not need to check other + # we do not need to check other return $rc fi else @@ -1221,6 +1262,70 @@ restart_client_loads () { fi done } + +# Start vmstat and save its process ID in a file. +start_vmstat() { + local nodes=$1 + local pid_file=$2 + + [ -z "$nodes" -o -z "$pid_file" ] && return 0 + + do_nodes $nodes \ + "vmstat 1 > $TESTLOG_PREFIX.$TESTNAME.vmstat.\\\$(hostname -s).log \ + 2>/dev/null $pid_file" +} + +# Display the nodes on which client loads failed. +print_end_run_file() { + local file=$1 + local node + + [ -s $file ] || return 0 + + echo "Found the END_RUN_FILE file: $file" + cat $file + + # A client load will stop if it finds the END_RUN_FILE file. + # That does not mean the client load actually failed though. + # The first node in END_RUN_FILE is the one we are interested in. + read node < $file + + if [ -n "$node" ]; then + local var=$(node_var_name $node)_load + + local prefix=$TESTLOG_PREFIX + [ -n "$TESTNAME" ] && prefix=$prefix.$TESTNAME + local stdout_log=$prefix.run_${!var}_stdout.$node.log + local debug_log=$(echo $stdout_log | sed 's/\(.*\)stdout/\1debug/') + + echo "Client load ${!var} failed on node $node:" + echo "$stdout_log" + echo "$debug_log" + fi +} + +# Stop the process which had its PID saved in a file. +stop_process() { + local nodes=$1 + local pid_file=$2 + + [ -z "$nodes" -o -z "$pid_file" ] && return 0 + + do_nodes $nodes "test -f $pid_file && + { kill -s TERM \\\$(cat $pid_file); rm -f $pid_file; }" || true +} + +# Stop all client loads. +stop_client_loads() { + local nodes=${1:-$CLIENTS} + local pid_file=$2 + + # stop the client loads + stop_process $nodes $pid_file + + # clean up the processes that started them + [ -n "$CLIENT_LOAD_PIDS" ] && kill -9 $CLIENT_LOAD_PIDS 2>/dev/null || true +} # End recovery-scale functions # verify that lustre actually cleaned up properly @@ -1255,7 +1360,8 @@ wait_update () { local RESULT local WAIT=0 - local sleep=5 + local sleep=1 + local print=10 while [ true ]; do RESULT=$(do_node $node "$TEST") if [ "$RESULT" == "$FINAL" ]; then @@ -1264,7 +1370,8 @@ wait_update () { return 0 fi [ $WAIT -ge $MAX ] && break - echo "Waiting $((MAX - WAIT)) secs for update" + [ $((WAIT % print)) -eq 0 ] && + echo "Waiting $((MAX - WAIT)) secs for update" WAIT=$((WAIT + sleep)) sleep $sleep done @@ -1925,6 +2032,7 @@ do_nodes() { [ -z "$myPDSH" -o "$myPDSH" = "no_dsh" -o "$myPDSH" = "rsh" ] && \ echo "cannot run remote command on $rnodes with $myPDSH" && return 128 + export FANOUT=$(get_node_count "${rnodes//,/ }") if $VERBOSE; then echo "CMD: $rnodes $@" >&2 $myPDSH $rnodes "$LCTL mark \"$@\"" > /dev/null 2>&1 || : @@ -2254,6 +2362,7 @@ setupall() { [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE mount_client $MOUNT [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT + clients_up if [ "$MOUNT_2" ]; then mount_client $MOUNT2 @@ -2361,12 +2470,6 @@ osc_ensure_active () { } init_param_vars () { - if ! remote_ost_nodsh && ! remote_mds_nodsh; then - export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2) - export OSTVER=$(do_facet ost1 "lctl get_param version" | cut -d. -f1,2) - export CLIVER=$(lctl get_param version | cut -d. -f 1,2) - fi - remote_mds_nodsh || TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") @@ -2697,7 +2800,8 @@ check_and_cleanup_lustre() { fi if is_mounted $MOUNT; then - [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* + [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || + error "remove sub-test dirs failed" [ "$ENABLE_QUOTA" ] && restore_quota_type || true fi @@ -3059,9 +3163,10 @@ error_noexit() { log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " + mkdir -p $LOGDIR # We need to dump the logs on all nodes if $dump; then - gather_logs $(comma_list $(nodes_list)) 0 + gather_logs $(comma_list $(nodes_list)) fi debugrestore @@ -3395,7 +3500,7 @@ check_grant() { # get server grant server_grant=`do_nodes $(comma_list $(osts_nodes)) \ - "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | \ + "$LCTL get_param -n obdfilter.${FSNAME}-OST*.tot_granted" | awk '{total += $1} END{print total}'` # check whether client grant == server grant @@ -3441,7 +3546,7 @@ index_from_ostuuid() mdtuuid_from_index() { - $LFS mdts $2 | awk '/^'$1'/ { print $2 }' + $LFS mdts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p" } remote_node () { @@ -3614,7 +3719,8 @@ is_patchless () } check_versions () { - [ "$MDSVER" = "$CLIVER" -a "$OSTVER" = "$CLIVER" ] + [ "$(lustre_version_code client)" = "$(lustre_version_code $SINGLEMDS)" -a \ + "$(lustre_version_code client)" = "$(lustre_version_code ost1)" ] } get_node_count() { @@ -3837,25 +3943,6 @@ exit \\\$rc;" fi } -# $1 node -# $2 file -# $3 $RUNAS -get_stripe_info() { - local tmp_file - - stripe_size=0 - stripe_count=0 - stripe_index=0 - tmp_file=$(mktemp) - - do_facet $1 $3 lfs getstripe -v $2 > $tmp_file - - stripe_size=`awk '$1 ~ /size/ {print $2}' $tmp_file` - stripe_count=`awk '$1 ~ /count/ {print $2}' $tmp_file` - stripe_index=`awk '$1 ~ /stripe_offset/ {print $2}' $tmp_file` - rm -f $tmp_file -} - # CMD: determine mds index where directory inode presents get_mds_dir () { local dir=$1 @@ -3919,19 +4006,12 @@ get_clientosc_proc_path() { get_lustre_version () { local facet=${1:-"$SINGLEMDS"} - do_facet $facet $LCTL get_param -n version | awk '/^lustre:/ {print $2}' -} - -get_mds_version_major () { - local facet=${1:-"$SINGLEMDS"} - local version=$(get_lustre_version $facet) - echo $version | awk -F. '{print $1}' + do_facet $facet $LCTL get_param -n version | awk '/^lustre:/ {print $2}' } -get_mds_version_minor () { +lustre_version_code() { local facet=${1:-"$SINGLEMDS"} - local version=$(get_lustre_version $facet) - echo $version | awk -F. '{print $2}' + version_code $(get_lustre_version $1) } # If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names @@ -3940,10 +4020,8 @@ get_mds_version_minor () { # mdt osc: fsname-OSTXXXX-osc mds_on_old_device() { local mds=${1:-"$SINGLEMDS"} - local major=$(get_mds_version_major $mds) - local minor=$(get_mds_version_minor $mds) - if [ $major -ge 2 ] || [ $major -eq 1 -a $minor -gt 8 ]; then + if [ $(lustre_version_code $mds) -gt $(version_code 1.9.0) ]; then do_facet $mds "lctl list_param osc.$FSNAME-OST*-osc \ > /dev/null 2>&1" && return 0 fi @@ -3958,9 +4036,8 @@ get_mdtosc_proc_path() { local mdt_label=$(convert_facet2label $mds_facet) local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//') - local major=$(get_mds_version_major $mds_facet) - local minor=$(get_mds_version_minor $mds_facet) - if [ $major -le 1 -a $minor -le 8 ] || mds_on_old_device $mds_facet; then + if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] || + mds_on_old_device $mds_facet; then echo "${ost_label}-osc" else echo "${ost_label}-osc-${mdt_index}" @@ -4046,10 +4123,7 @@ wait_import_state() { # the value depends on configure options, and it is not stored in /proc. # obd_support.h: # #define CONNECTION_SWITCH_MIN 5U -# #ifndef CRAY_XT3 # #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20) -# #else -# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/2) request_timeout () { local facet=$1 @@ -4228,7 +4302,7 @@ destroy_pools () { echo destroy the created pools: ${!listvar} for poolname in ${!listvar//,/ }; do - destroy_pool $fsname.$poolname + destroy_pool $fsname.$poolname done } @@ -4240,15 +4314,21 @@ cleanup_pools () { gather_logs () { local list=$1 - local tar_logs=$2 local ts=$(date +%s) local docp=true + + if [[ ! -f "$YAML_LOG" ]]; then + # init_logging is not performed before gather_logs, + # so the $LOGDIR needs to be checked here + check_shared_dir $LOGDIR && touch $LOGDIR/shared + fi + [ -f $LOGDIR/shared ] && docp=false - + # dump lustre logs, dmesg - prefix="$LOGDIR/${TESTSUITE}.${TESTNAME}" + prefix="$TESTLOG_PREFIX.$TESTNAME" suffix="$ts.log" echo "Dumping lctl log to ${prefix}.*.${suffix}" @@ -4264,22 +4344,9 @@ gather_logs () { dmesg > ${prefix}.dmesg.\\\$(hostname -s).${suffix}" if [ ! -f $LOGDIR/shared ]; then do_nodes $list rsync -az "${prefix}.*.${suffix}" $HOSTNAME:$LOGDIR - fi - - if [ $tar_logs == 1 ]; then - local archive=$LOGDIR/${TESTSUITE}-$ts.tar.bz2 - tar -jcf $archive $LOGDIR/*$ts* $LOGDIR/*${TESTSUITE}* - - echo $archive fi } -cleanup_logs () { - local list=${1:-$(comma_list $(nodes_list))} - - [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true -} - do_ls () { local mntpt_root=$1 local num_mntpts=$2 @@ -4622,20 +4689,24 @@ check_logdir() { # Not found. Create local logdir mkdir -p $dir else - touch $dir/node.$(hostname -s).yml + touch $dir/check_file.$(hostname -s) fi return 0 } check_write_access() { local dir=$1 + local node + local file + for node in $(nodes_list); do - if [ ! -f "$dir/node.$(short_hostname ${node}).yml" ]; then + file=$dir/check_file.$(short_hostname $node) + if [[ ! -f "$file" ]]; then # Logdir not accessible/writable from this node. return 1 fi + rm -f $file || return 1 done - rm -f $dir/node.*.yml return 0 } @@ -4650,15 +4721,17 @@ init_logging() { mkdir -p $LOGDIR init_clients_lists - if check_shared_dir $LOGDIR; then - touch $LOGDIR/shared - echo "Logging to shared log directory: $LOGDIR" - else - echo "Logging to local directory: $LOGDIR" - fi + if [ ! -f $YAML_LOG ]; then # If the yaml log already exists then we will just append to it + if check_shared_dir $LOGDIR; then + touch $LOGDIR/shared + echo "Logging to shared log directory: $LOGDIR" + else + echo "Logging to local directory: $LOGDIR" + fi - yml_nodes_file $LOGDIR >> $YAML_LOG - yml_results_file >> $YAML_LOG + yml_nodes_file $LOGDIR >> $YAML_LOG + yml_results_file >> $YAML_LOG + fi umask $SAVE_UMASK }