X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=a690b1c50f73d5da0392247a40b33f6e33d13e04;hp=cf13aed925150149a370ebbfaf90eafbb74a13e0;hb=088f78e6ed20210c0ed265ff4bca6063e316bd90;hpb=6c80ca6ae2fdecc8653b8309b61d602c2746033d diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index cf13aed..a690b1c 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1,15 +1,14 @@ #!/bin/bash # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: -trap 'print_summary && echo "test-framework exiting on error"' ERR +trap 'print_summary && touch $TF_FAIL && \ + echo "test-framework exiting on error"' ERR set -e #set -x - export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} export VERBOSE=false -export GMNALNID=${GMNALNID:-/usr/sbin/gmlndnid} export CATASTROPHE=${CATASTROPHE:-/proc/sys/lnet/catastrophe} export GSS=false export GSS_KRB5=false @@ -34,6 +33,9 @@ if [ -f "$EXCEPT_LIST_FILE" ]; then . $EXCEPT_LIST_FILE fi +[ -z "$MODPROBECONF" -a -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf +[ -z "$MODPROBECONF" -a -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre + assert_DIR () { local failed="" [[ $DIR/ = $MOUNT/* ]] || \ @@ -108,7 +110,22 @@ init_test_env() { export E2LABEL=${E2LABEL:-e2label} export DUMPE2FS=${DUMPE2FS:-dumpe2fs} export E2FSCK=${E2FSCK:-e2fsck} + export LFSCK_BIN=${LFSCK_BIN:-lfsck} + export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check filesystem after each test suit + export SKIP_LFSCK=${SKIP_LFSCK:-"yes"} # bug 13698, change to "no" when fixed + export SHARED_DIRECTORY=${SHARED_DIRECTORY:-"/tmp"} + export FSCK_MAX_ERR=4 # File system errors left uncorrected + if [ "$SKIP_LFSCK" == "no" ]; then + if [ ! -x `which $LFSCK_BIN` ]; then + log "$($E2FSCK -V)" + error_exit "$E2FSCK does not support lfsck" + fi + export MDSDB=${MDSDB:-$SHARED_DIRECTORY/mdsdb} + export OSTDB=${OSTDB:-$SHARED_DIRECTORY/ostdb} + export MDSDB_OPT="--mdsdb $MDSDB" + export OSTDB_OPT="--ostdb $OSTDB-\$ostidx" + fi #[ -d /r ] && export ROOT=${ROOT:-/r} export TMP=${TMP:-$ROOT/tmp} export TESTSUITELOG=${TMP}/${TESTSUITE}.log @@ -124,7 +141,7 @@ init_test_env() { export PATH=$PATH:$LUSTRE/utils/gss fi if ! echo $PATH | grep -q $LUSTRE/tests; then - export PATH=$PATH:$LUSTRE/tests + export PATH=$LUSTRE/tests:$PATH fi export LST=${LST:-"$LUSTRE/../lnet/utils/lst"} [ ! -f "$LST" ] && export LST=$(which lst) @@ -149,6 +166,8 @@ init_test_env() { export L_GETIDENTITY=NONE fi fi + export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"} + [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID=$(which ll_decode_filter_fid) export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"} [ ! -f "$MKFS" ] && export MKFS=$(which mkfs.lustre) export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} @@ -230,7 +249,16 @@ module_loaded () { /sbin/lsmod | grep -q $1 } +# Load a module on the system where this is running. +# +# Synopsis: load_module module_name [module arguments for insmod/modprobe] +# +# If module arguments are not given but MODOPTS_ is set, then its value +# will be used as the arguments. Otherwise arguments will be obtained from +# /etc/modprobe.conf, from /etc/modprobe.d/Lustre, or else none will be used. +# load_module() { + local optvar EXT=".ko" module=$1 shift @@ -238,19 +266,47 @@ load_module() { module_loaded ${BASE} && return + # If no module arguments were passed, get them from $MODOPTS_, else from + # modprobe.conf + if [ $# -eq 0 ]; then + # $MODOPTS_; we could use associative arrays, but that's not in + # Bash until 4.x, so we resort to eval. + optvar="MODOPTS_$(basename $module | tr a-z A-Z)" + eval set -- \$$optvar + if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then + # Nothing in $MODOPTS_; try modprobe.conf + set -- $(grep "^options\\s*\<${module}\>" $MODPROBECONF) + # Get rid of "options $module" + (($# > 0)) && shift 2 + + # Ensure we have accept=all for lnet + if [ $(basename $module) = lnet ]; then + # OK, this is a bit wordy... + local arg accept_all_present=false + for arg in "$@"; do + [ "$arg" = accept=all ] && accept_all_present=true + done + $accept_all_present || set -- "$@" accept=all + fi + fi + fi + + [ $# -gt 0 ] && echo "${module} options: '$*'" + + # Note that insmod will ignore anything in modprobe.conf, which is why we're + # passing options on the command-line. if [ "$BASE" == "lnet_selftest" ] && \ [ -f ${LUSTRE}/../lnet/selftest/${module}${EXT} ]; then insmod ${LUSTRE}/../lnet/selftest/${module}${EXT} - elif [ -f ${LUSTRE}/${module}${EXT} ]; then - insmod ${LUSTRE}/${module}${EXT} $@ + insmod ${LUSTRE}/${module}${EXT} "$@" else # must be testing a "make install" or "rpm" installation # note failed to load ptlrpc_gss is considered not fatal if [ "$BASE" == "ptlrpc_gss" ]; then - modprobe $BASE $@ 2>/dev/null || echo "gss/krb5 is not supported" + modprobe $BASE "$@" 2>/dev/null || echo "gss/krb5 is not supported" else - modprobe $BASE $@ + modprobe $BASE "$@" fi fi } @@ -258,34 +314,22 @@ load_module() { load_modules_local() { if [ -n "$MODPROBE" ]; then # use modprobe - return 0 - fi - if [ "$HAVE_MODULES" = true ]; then - # we already loaded + echo "Using modprobe to load modules" return 0 fi - HAVE_MODULES=true echo Loading modules from $LUSTRE load_module ../libcfs/libcfs/libcfs [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" - local MODPROBECONF= - [ -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf - [ ! "$MODPROBECONF" -a -d /etc/modprobe.d ] && MODPROBECONF=/etc/modprobe.d/Lustre - [ -z "$LNETOPTS" -a "$MODPROBECONF" ] && \ - LNETOPTS=$(awk '/^options lnet/ { print $0}' $MODPROBECONF | sed 's/^options lnet //g') - echo $LNETOPTS | grep -q "accept=all" || LNETOPTS="$LNETOPTS accept=all"; - echo "lnet options: '$LNETOPTS'" - # note that insmod will ignore anything in modprobe.conf - load_module ../lnet/lnet/lnet $LNETOPTS + load_module ../lnet/lnet/lnet LNETLND=${LNETLND:-"socklnd/ksocklnd"} load_module ../lnet/klnds/$LNETLND load_module lvfs/lvfs load_module obdclass/obdclass load_module ptlrpc/ptlrpc load_module ptlrpc/gss/ptlrpc_gss - [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota + [ "$USE_QUOTA" = "yes" -a "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS load_module fld/fld load_module fid/fid load_module lmv/lmv @@ -303,7 +347,7 @@ load_modules_local() { load_module mdt/mdt load_module lvfs/fsfilt_$FSTYPE load_module cmm/cmm - load_module osd/osd + load_module osd-ldiskfs/osd_ldiskfs load_module ost/ost load_module obdfilter/obdfilter fi @@ -358,8 +402,6 @@ unload_modules() { fi fi - HAVE_MODULES=false - check_mem_leak || return 254 echo "modules unloaded." @@ -370,7 +412,7 @@ check_gss_daemon_nodes() { local list=$1 dname=$2 - do_nodes --verbose $list "num=\\\$(ps -o cmd -C $dname | grep $dname | wc -l); + do_nodesv $list "num=\\\$(ps -o cmd -C $dname | grep $dname | wc -l); if [ \\\"\\\$num\\\" -ne 1 ]; then echo \\\$num instance of $dname; exit 1; @@ -852,6 +894,13 @@ shutdown_facet() { fi } +remount_facet() { + local facet=$1 + + stop $facet + mount_facet $facet +} + reboot_facet() { facet=$1 if [ "$FAILURE_MODE" = HARD ]; then @@ -916,6 +965,8 @@ start_client_loads () { testnum=$((nodenum % numloads)) start_client_load ${clients[nodenum]} ${CLIENT_LOADS[testnum]} done + # bug 22169: wait the background threads to start + sleep 2 } # only for remote client @@ -1101,10 +1152,7 @@ wait_recovery_complete () { local facet=$1 # Use default policy if $2 is not passed by caller. - #define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2) - # as we are in process of changing obd_timeout in different ways - # let's set MAX longer than that - local MAX=${2:-$(( TIMEOUT * 4 ))} + local MAX=${2:-$(max_recovery_time)} local var_svc=${facet}_svc local procfile="*.${!var_svc}.recovery_status" @@ -1133,6 +1181,7 @@ wait_mds_ost_sync () { local -a sync=($(do_nodes $(comma_list $(osts_nodes)) \ "$LCTL get_param -n obdfilter.*.mds_sync")) local con=1 + local i for ((i=0; i<${#sync[@]}; i++)); do [ ${sync[$i]} -eq 0 ] && continue # there is a not finished MDS-OST synchronization @@ -1348,13 +1397,6 @@ do_lmc() { exit 1 } -h2gm () { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - ID=`$PDSH $1 $GMNALNID -l | cut -d\ -f2` - echo $ID"@gm" - fi -} - h2name_or_ip() { if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else echo $1"@$2" @@ -1501,10 +1543,25 @@ do_node() { return ${PIPESTATUS[0]} } +do_nodev() { + do_node --verbose "$@" +} + single_local_node () { [ "$1" = "$HOSTNAME" ] } +# Outputs environment variable assignments that should be passed to remote nodes +get_env_vars() { + local var + local value + + for var in ${!MODOPTS_*}; do + value=${!var} + echo "${var}=\"$value\"" + done +} + do_nodes() { local verbose=false # do not stripe off hostname if verbose, bug 19215 @@ -1518,9 +1575,9 @@ do_nodes() { if single_local_node $rnodes; then if $verbose; then - do_node --verbose $rnodes $@ + do_nodev $rnodes "$@" else - do_node $rnodes $@ + do_node $rnodes "$@" fi return $? fi @@ -1536,10 +1593,12 @@ do_nodes() { $myPDSH $rnodes $LCTL mark "$@" > /dev/null 2>&1 || : fi - if $verbose ; then - $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\")" + # do not replace anything from pdsh output if -N is used + # -N Disable hostname: prefix on lines of output. + if $verbose || [[ $myPDSH = *-N* ]]; then + $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")" else - $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\")" | sed -re "s/\w+:\s//g" + $myPDSH $rnodes "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; cd $RPWD; LUSTRE=\"$RLUSTRE\" $(get_env_vars) sh -c \"$@\")" | sed -re "s/^[^:]*: //g" fi return ${PIPESTATUS[0]} } @@ -1552,6 +1611,10 @@ do_facet() { do_node $HOST "$@" } +do_nodesv() { + do_nodes --verbose "$@" +} + add() { local facet=$1 shift @@ -1608,13 +1671,28 @@ stopall() { rm -f $TMP/ost${num}active done + if ! combined_mgs_mds ; then + stop mgs + fi + return 0 } +cleanup_echo_devs () { + local devs=$($LCTL dl | grep echo | awk '{print $4}') + + for dev in $devs; do + $LCTL --device $dev cleanup + $LCTL --device $dev detach + done +} + cleanupall() { nfs_client_mode && return stopall $* + cleanup_echo_devs + unload_modules cleanup_gss } @@ -1625,6 +1703,10 @@ mdsmkfsopts() test $nr = 1 && echo -n $MDS_MKFS_OPTS || echo -n $MDSn_MKFS_OPTS } +combined_mgs_mds () { + [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]] +} + formatall() { if [ "$IAMDIR" == "yes" ]; then MDS_MKFS_OPTS="$MDS_MKFS_OPTS --iam-dir" @@ -1638,7 +1720,7 @@ formatall() { load_modules [ "$CLIENTONLY" ] && return echo Formatting mgs, mds, osts - if [[ $MDSDEV1 != $MGSDEV ]] || [[ $mds1_HOST != $mgs_HOST ]]; then + if ! combined_mgs_mds ; then add mgs $mgs_MKFS_OPTS $FSTYPE_OPT --reformat $MGSDEV || exit 10 fi @@ -1691,7 +1773,7 @@ switch_identity() { do_facet mds$num "lctl set_param -n mdt.$MDT.identity_upcall \"NONE\"" fi - do_facet mds$num "lctl set_param -n mdt/$MDT/identity_flush \"-1\"" + do_facet mds$num "lctl set_param -n mdt/$MDT/identity_flush=-1" if [ $old = "NONE" ]; then return 1 @@ -1737,7 +1819,7 @@ setupall() { echo Setup mgs, mdt, osts echo $WRITECONF | grep -q "writeconf" && \ writeconf_all - if [[ $mds1_HOST != $mgs_HOST ]] || [[ $MDSDEV1 != $MGSDEV ]]; then + if ! combined_mgs_mds ; then start mgs $MGSDEV $mgs_MOUNT_OPTS fi @@ -1857,12 +1939,11 @@ init_facets_vars () { osc_ensure_active () { local facet=$1 - local type=$2 - local timeout=$3 + local timeout=$2 local period=0 while [ $period -lt $timeout ]; do - count=$(do_facet $facet "lctl dl | grep '${FSNAME}-OST.*-osc-${type}' | grep ' IN ' 2>/dev/null | wc -l") + count=$(do_facet $facet "lctl dl | grep ' IN osc ' 2>/dev/null | wc -l") if [ $count -eq 0 ]; then break fi @@ -1875,11 +1956,6 @@ osc_ensure_active () { [ $period -lt $timeout ] || log "$count OST are inactive after $timeout seconds, give up" } -som_check() { - SOM_ENABLED=$(do_facet $SINGLEMDS "$LCTL get_param mdt.*.som" | awk -F= ' {print $2}' | head -n 1) - echo $SOM_ENABLED -} - init_param_vars () { if ! remote_ost_nodsh && ! remote_mds_nodsh; then export MDSVER=$(do_facet $SINGLEMDS "lctl get_param version" | cut -d. -f1,2) @@ -1892,13 +1968,9 @@ init_param_vars () { log "Using TIMEOUT=$TIMEOUT" - osc_ensure_active $SINGLEMDS M $TIMEOUT - osc_ensure_active client c $TIMEOUT + osc_ensure_active $SINGLEMDS $TIMEOUT + osc_ensure_active client $TIMEOUT - if [ x"$(som_check)" = x"enabled" ]; then - ENABLE_QUOTA="" - echo "disable quota temporary when SOM enabled" - fi if [ $QUOTA_AUTO -ne 0 ]; then if [ "$ENABLE_QUOTA" ]; then echo "enable quota as required" @@ -1992,11 +2064,26 @@ check_timeout () { is_mounted () { local mntpt=$1 + [ -z $mntpt ] && return 1 local mounted=$(mounted_lustre_filesystems) echo $mounted' ' | grep -w -q $mntpt' ' } +is_empty_dir() { + [ $(find $1 -maxdepth 1 -print | wc -l) = 1 ] && return 0 + return 1 +} + +# empty lustre filesystem may have empty directories lost+found and .lustre +is_empty_fs() { + [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \ + -print | wc -l) = 1 ] || return 1 + [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found && return 0 + [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre && return 0 + return 1 +} + check_and_setup_lustre() { nfs_client_mode && return @@ -2091,7 +2178,121 @@ cleanup_and_setup_lustre() { check_and_setup_lustre } +# Get all of the server target devices from a given server node and type. +get_mnt_devs() { + local node=$1 + local type=$2 + local obd_type + local devs + local dev + + case $type in + mdt) obd_type="osd" ;; + ost) obd_type="obdfilter" ;; # needs to be fixed when OST also uses an OSD + *) echo "invalid server type" && return 1 ;; + esac + + devs=$(do_node $node "lctl get_param -n $obd_type*.*.mntdev") + for dev in $devs; do + case $dev in + *loop*) do_node $node "losetup $dev" | \ + sed -e "s/.*(//" -e "s/).*//" ;; + *) echo $dev ;; + esac + done +} + +# Get all of the server target devices. +get_svr_devs() { + local i + + # MDT device + MDTDEV=$(get_mnt_devs $(mdts_nodes) mdt) + + # OST devices + i=0 + for node in $(osts_nodes); do + OSTDEVS[i]=$(get_mnt_devs $node ost) + i=$((i + 1)) + done +} + +# Run e2fsck on MDT or OST device. +run_e2fsck() { + local node=$1 + local target_dev=$2 + local ostidx=$3 + local ostdb_opt=$4 + + df > /dev/null # update statfs data on disk + local cmd="$E2FSCK -d -v -f -n $MDSDB_OPT $ostdb_opt $target_dev" + echo $cmd + local rc=0 + do_node $node $cmd || rc =$? + [ $rc -le $FSCK_MAX_ERR ] || \ + error "$cmd returned $rc, should be <= $FSCK_MAX_ERR" + return 0 +} + +# Run e2fsck on MDT and OST(s) to generate databases used for lfsck. +generate_db() { + local i + local ostidx + local dev + local tmp_file + + [ $MDSCOUNT -eq 1 ] || error "CMD is not supported" + tmp_file=$(mktemp -p $SHARED_DIRECTORY || + error "fail to create file in $SHARED_DIRECTORY") + + # make sure everything gets to the backing store + local list=$(comma_list $CLIENTS $(facet_host $SINGLEMDS) $(osts_nodes)) + do_nodes $list "sync; sleep 2; sync" + + do_nodes $list ls $tmp_file || \ + error "$SHARED_DIRECTORY is not a shared directory" + rm $tmp_file + + run_e2fsck $(mdts_nodes) $MDTDEV + + i=0 + ostidx=0 + OSTDB_LIST="" + for node in $(osts_nodes); do + for dev in ${OSTDEVS[i]}; do + local ostdb_opt=`eval echo $OSTDB_OPT` + run_e2fsck $node $dev $ostidx "$ostdb_opt" + OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx" + ostidx=$((ostidx + 1)) + done + i=$((i + 1)) + done +} + +run_lfsck() { + local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT" + echo $cmd + local rc=0 + eval $cmd || rc=$? + [ $rc -le $FSCK_MAX_ERR ] || \ + error "$cmd returned $rc, should be <= $FSCK_MAX_ERR" + echo "lfsck finished with rc=$rc" + + rm -rvf $MDSDB* $OSTDB* || true + return 0 +} + check_and_cleanup_lustre() { + if [ "$LFSCK_ALWAYS" = "yes" ]; then + get_svr_devs + generate_db + if [ "$SKIP_LFSCK" == "no" ]; then + run_lfsck + else + echo "skip lfsck" + fi + fi + if is_mounted $MOUNT; then [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* [ "$ENABLE_QUOTA" ] && restore_quota_type || true @@ -2367,10 +2568,6 @@ pgcache_empty() { return 1 fi done - if [[ $MDSDEV1 != $MGSDEV ]]; then - stop mgs - fi - return 0 } @@ -2438,15 +2635,11 @@ error_noexit() { error() { error_noexit "$@" - if $FAIL_ON_ERROR; then - reset_fail_loc - exit 1 - fi + exit 1 } error_exit() { - error_noexit "$@" - exit 1 + error "$@" } # use only if we are ignoring failures for this test, bugno required. @@ -2501,6 +2694,14 @@ basetest() { # print a newline if the last test was skipped export LAST_SKIPPED= +# +# Main entry into test-framework. This is called with the name and +# description of a test. The name is used to find the function to run +# the test using "test_$name". +# +# This supports a variety of methods of specifying specific test to +# run or not run. These need to be documented... +# run_test() { assert_DIR @@ -2553,13 +2754,8 @@ run_test() { return $? } -EQUALS="======================================================================" equals_msg() { - msg="$@" - - local suffixlen=$((${#EQUALS} - ${#msg})) - [ $suffixlen -lt 5 ] && suffixlen=5 - log `echo $(printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS)` + banner "$*" } log() { @@ -2609,17 +2805,38 @@ reset_fail_loc () { echo done. } + +# +# Log a message (on all nodes) padded with "=" before and after. +# Also appends a timestamp and prepends the testsuite name. +# + +EQUALS="====================================================================================================" +banner() { + msg="== ${TESTSUITE} $*" + last=${msg: -1:1} + [[ $last != "=" && $last != " " ]] && msg="$msg " + msg=$(printf '%s%.*s' "$msg" $((${#EQUALS} - ${#msg})) $EQUALS ) + # always include at least == after the message + log "$msg== $(date +"%H:%M:%S (%s)")" +} + +# +# Run a single test function and cleanup after it. +# +# This function should be run in a subshell so the test func can +# exit() without stopping the whole script. +# run_one() { local testnum=$1 local message=$2 - local start_tm=$3 tfile=f${testnum} export tdir=d0.${TESTSUITE}/d${base} export TESTNAME=test_$testnum local SAVE_UMASK=`umask` umask 0022 - log "== test $testnum: $message == `date +%H:%M:%S` ($start_tm)" + banner "test $testnum: $message" test_${testnum} || error "test_$testnum failed with $?" cd $SAVE_PWD reset_fail_loc @@ -2632,6 +2849,12 @@ run_one() { return 0 } +# +# Wrapper around run_one to ensure: +# - test runs in subshell +# - output of test is saved to separate log file for error reporting +# - test result is saved to data file +# run_one_logged() { local BEFORE=`date +%s` local TEST_ERROR @@ -2640,7 +2863,8 @@ run_one_logged() { rm -rf $LOGDIR/err echo - run_one $1 "$2" $BEFORE 2>&1 | tee $test_log + log_sub_test_begin test_${1} + (run_one $1 "$2") 2>&1 | tee $test_log local RC=${PIPESTATUS[0]} [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \ @@ -2649,7 +2873,7 @@ run_one_logged() { duration=$((`date +%s` - $BEFORE)) pass "(${duration}s)" [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err) - log_sub_test test_${1} $TEST_STATUS $duration "$RC" "$TEST_ERROR" + log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR" if [ -f $LOGDIR/err ]; then $FAIL_ON_ERROR && exit $RC @@ -2905,9 +3129,10 @@ mixed_mdt_devs () { generate_machine_file() { local nodes=${1//,/ } local machinefile=$2 - rm -f $machinefile || error "can't rm $machinefile" + rm -f $machinefile for node in $nodes; do - echo $node >>$machinefile + echo $node >>$machinefile || \ + { echo "can not generate machinefile $machinefile" && return 1; } done } @@ -2918,6 +3143,17 @@ get_stripe () { rm -f $file } +setstripe_nfsserver () { + local dir=$1 + + local nfsserver=$(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \ + { print $1 }' /proc/mounts | cut -f 1 -d : | head -1) + + [ -z $nfsserver ] && echo "$dir is not nfs mounted" && return 1 + + do_nodev $nfsserver lfs setstripe "$@" +} + check_runas_id_ret() { local myRC=0 local myRUNAS_UID=$1 @@ -3026,6 +3262,10 @@ inodes_available () { echo $IFree } +mdsrate_inodes_available () { + echo $(($(inodes_available) - 1)) +} + # reset llite stat counters clear_llite_stats(){ lctl set_param -n llite.*.stats 0 @@ -3063,7 +3303,7 @@ calc_osc_kbytes () { # generate a stream of formatted strings ( =) save_lustre_params() { local s - do_nodes --verbose $1 "lctl get_param $2 | while read s; do echo \\\$s; done" + do_nodesv $1 "lctl get_param $2 | while read s; do echo \\\$s; done" } # restore lustre parameters from input stream, produces by save_lustre_params @@ -3171,28 +3411,52 @@ get_clientosc_proc_path() { } get_lustre_version () { - local node=${1:-"mds"} - do_facet $node $LCTL get_param -n version | awk '/^lustre:/ {print $2}' + local facet=${1:-"$SINGLEMDS"} + do_facet $facet $LCTL get_param -n version | awk '/^lustre:/ {print $2}' } get_mds_version_major () { - local version=$(get_lustre_version mds) + local facet=${1:-"$SINGLEMDS"} + local version=$(get_lustre_version $facet) echo $version | awk -F. '{print $1}' } get_mds_version_minor () { - local version=$(get_lustre_version mds) + local facet=${1:-"$SINGLEMDS"} + local version=$(get_lustre_version $facet) echo $version | awk -F. '{print $2}' } +# If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names +# used by MDT would not be changed. +# mdt lov: fsname-mdtlov +# mdt osc: fsname-OSTXXXX-osc +mds_on_old_device() { + local mds=${1:-"$SINGLEMDS"} + local major=$(get_mds_version_major $mds) + local minor=$(get_mds_version_minor $mds) + + if [ $major -ge 2 ] || [ $major -eq 1 -a $minor -gt 8 ]; then + do_facet $mds "lctl list_param osc.$FSNAME-OST*-osc \ + > /dev/null 2>&1" && return 0 + fi + return 1 +} + get_mdtosc_proc_path() { - local ost=$1 - local major=$(get_mds_version_major) - local minor=$(get_mds_version_minor) - if [ $major -le 1 -a $minor -le 8 ] ; then - echo "${ost}-osc" + local mds_facet=$1 + local ost_label=${2:-"*OST*"} + + [ "$mds_facet" = "mds" ] && mds_facet=$SINGLEMDS + local mdt_label=$(convert_facet2label $mds_facet) + local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//') + + local major=$(get_mds_version_major $mds_facet) + local minor=$(get_mds_version_minor $mds_facet) + if [ $major -le 1 -a $minor -le 8 ] || mds_on_old_device $mds_facet; then + echo "${ost_label}-osc" else - echo "${ost}-osc-MDT0000" + echo "${ost_label}-osc-${mdt_index}" fi } @@ -3201,8 +3465,8 @@ get_osc_import_name() { local ost=$2 local label=$(convert_facet2label $ost) - if [ "$facet" == "mds" ]; then - get_mdtosc_proc_path $label + if [ "${facet:0:3}" = "mds" ]; then + get_mdtosc_proc_path $facet $label return 0 fi @@ -3277,7 +3541,7 @@ do_rpc_nodes () { # Add paths to lustre tests for 32 and 64 bit systems. local RPATH="$RLUSTRE/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:$PATH" - do_nodes --verbose $list "PATH=$RPATH sh rpc.sh $@ " + do_nodesv $list "PATH=$RPATH sh rpc.sh $@ " } wait_clients_import_state () { @@ -3438,7 +3702,7 @@ gather_logs () { return fi - do_nodes --verbose $list \ + do_nodesv $list \ "$LCTL dk > ${prefix}.debug_log.\\\$(hostname).${suffix}; dmesg > ${prefix}.dmesg.\\\$(hostname).${suffix}" if [ ! -f $LOGDIR/shared ]; then @@ -3480,6 +3744,24 @@ do_ls () { return $rc } +# target_start_and_reset_recovery_timer() +# service_time = at_est2timeout(service_time); +# service_time += 2 * (CONNECTION_SWITCH_MAX + CONNECTION_SWITCH_INC + +# INITIAL_CONNECT_TIMEOUT); +# CONNECTION_SWITCH_MAX : min(25U, max(CONNECTION_SWITCH_MIN,obd_timeout)) +#define CONNECTION_SWITCH_INC 1 +#define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20) +#define CONNECTION_SWITCH_MIN 5U + +max_recovery_time () { + local init_connect_timeout=$(( TIMEOUT / 20 )) + [[ $init_connect_timeout > 5 ]] || init_connect_timeout=5 + + local service_time=$(( $(at_max_get client) + $(( 2 * $(( 25 + 1 + init_connect_timeout)) )) )) + + echo $service_time +} + get_clients_mount_count () { local clients=${CLIENTS:-`hostname`} @@ -3501,7 +3783,7 @@ combination() R=0 else N=$((N + 1)) - while [ $N -le $M ]; do + while [ $N -lt $M ]; do R=$((R * N)) N=$((N + 1)) done @@ -3646,9 +3928,13 @@ flvr_cnt_mdt2ost() { local flavor=$1 local cnt=0 + local mdtosc for num in `seq $MDSCOUNT`; do - output=`do_facet mds$num lctl get_param -n osc.*OST*-osc-MDT*.$PROC_CLI 2>/dev/null` + mdtosc=$(get_mdtosc_proc_path mds$num) + mdtosc=${mdtosc/-MDT*/-MDT\*} + output=$(do_facet mds$num lctl get_param -n \ + osc.$mdtosc.$PROC_CLI 2>/dev/null) tmpcnt=`count_flvr "$output" $flavor` cnt=$((cnt + tmpcnt)) done @@ -3712,6 +3998,7 @@ wait_flavor() return 0 else echo "found $res $flavor connections of $dir, not ready ($expect)" + return 0 sleep 4 fi done @@ -3821,7 +4108,81 @@ log_test() { yml_log_test $1 >> $YAML_LOG } -log_sub_test() { - yml_log_sub_test $@ >> $YAML_LOG +log_test_status() { + yml_log_test_status $@ >> $YAML_LOG +} + +log_sub_test_begin() { + yml_log_sub_test_begin $@ >> $YAML_LOG +} + +log_sub_test_end() { + yml_log_sub_test_end $@ >> $YAML_LOG } +run_llverdev() +{ + local dev=$1 + local devname=$(basename $1) + local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}') + # loop devices aren't in /proc/partitions + [ "x$size" == "x" ] && local size=$(ls -l $dev | awk '{print $5}') + + size=$(($size / 1024 / 1024)) # Gb + + local partial_arg="" + # Run in partial (fast) mode if the size + # of a partition > 10 GB + [ $size -gt 10 ] && partial_arg="-p" + + llverdev --force $partial_arg $dev +} + +remove_mdt_files() { + local facet=$1 + local mdtdev=$2 + shift 2 + local files="$@" + local mntpt=${MOUNT%/*}/$facet + + echo "removing files from $mdtdev on $facet: $files" + mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $? + rc=0; + for f in $files; do + rm $mntpt/ROOT/$f || { rc=$?; break; } + done + umount -f $mntpt || return $? + return $rc +} + +duplicate_mdt_files() { + local facet=$1 + local mdtdev=$2 + shift 2 + local files="$@" + local mntpt=${MOUNT%/*}/$facet + + echo "duplicating files on $mdtdev on $facet: $files" + mkdir -p $mntpt || return $? + mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $? + + do_umount() { + trap 0 + popd > /dev/null + rm $tmp + umount -f $mntpt + } + trap do_umount EXIT + + tmp=$(mktemp $TMP/setfattr.XXXXXXXXXX) + pushd $mntpt/ROOT > /dev/null || return $? + rc=0 + for f in $files; do + touch $f.bad || return $? + getfattr -n trusted.lov $f | sed "s#$f#&.bad#" > $tmp + rc=${PIPESTATUS[0]} + [ $rc -eq 0 ] || return $rc + setfattr --restore $tmp || return $? + done + do_umount +}