#!/bin/bash
# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
-trap 'print_summary && echo "test-framework exiting on error"' ERR
+trap 'print_summary && touch $TF_FAIL && \
+ echo "test-framework exiting on error"' ERR
set -e
#set -x
-
export REFORMAT=${REFORMAT:-""}
export WRITECONF=${WRITECONF:-""}
export VERBOSE=false
export E2LABEL=${E2LABEL:-e2label}
export DUMPE2FS=${DUMPE2FS:-dumpe2fs}
export E2FSCK=${E2FSCK:-e2fsck}
+ export LFSCK_BIN=${LFSCK_BIN:-lfsck}
+ export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check filesystem after each test suit
+ export SKIP_LFSCK=${SKIP_LFSCK:-"yes"} # bug 13698, change to "no" when fixed
+ export SHARED_DIRECTORY=${SHARED_DIRECTORY:-"/tmp"}
+ export FSCK_MAX_ERR=4 # File system errors left uncorrected
+ if [ "$SKIP_LFSCK" == "no" ]; then
+ if [ ! -x `which $LFSCK_BIN` ]; then
+ log "$($E2FSCK -V)"
+ error_exit "$E2FSCK does not support lfsck"
+ fi
+ export MDSDB=${MDSDB:-$SHARED_DIRECTORY/mdsdb}
+ export OSTDB=${OSTDB:-$SHARED_DIRECTORY/ostdb}
+ export MDSDB_OPT="--mdsdb $MDSDB"
+ export OSTDB_OPT="--ostdb $OSTDB-\$ostidx"
+ fi
#[ -d /r ] && export ROOT=${ROOT:-/r}
export TMP=${TMP:-$ROOT/tmp}
export TESTSUITELOG=${TMP}/${TESTSUITE}.log
shift
BASE=`basename $module $EXT`
+ module_loaded ${BASE} && return
+
# If no module arguments were passed, get them from $MODOPTS_<MODULE>, else from
# modprobe.conf
if [ $# -eq 0 ]; then
# Nothing in $MODOPTS_<MODULE>; try modprobe.conf
set -- $(grep "^options\\s*\<${module}\>" $MODPROBECONF)
# Get rid of "options $module"
- (($# > 0)) && shift 2
+ (($# > 0)) && shift 2
# Ensure we have accept=all for lnet
- if [ $module = lnet ]; then
+ if [ $(basename $module) = lnet ]; then
# OK, this is a bit wordy...
local arg accept_all_present=false
for arg in "$@"; do
[ $# -gt 0 ] && echo "${module} options: '$*'"
- module_loaded ${BASE} && return
-
# Note that insmod will ignore anything in modprobe.conf, which is why we're
# passing options on the command-line.
if [ "$BASE" == "lnet_selftest" ] && \
echo "Using modprobe to load modules"
return 0
fi
- if [ "$HAVE_MODULES" = true ]; then
- # we already loaded
- echo "Modules already loaded"
- return 0
- fi
- HAVE_MODULES=true
echo Loading modules from $LUSTRE
load_module ../libcfs/libcfs/libcfs
fi
fi
- HAVE_MODULES=false
-
check_mem_leak || return 254
echo "modules unloaded."
local list=$1
dname=$2
- do_nodes --verbose $list "num=\\\$(ps -o cmd -C $dname | grep $dname | wc -l);
+ do_nodesv $list "num=\\\$(ps -o cmd -C $dname | grep $dname | wc -l);
if [ \\\"\\\$num\\\" -ne 1 ]; then
echo \\\$num instance of $dname;
exit 1;
fi
}
+remount_facet() {
+ local facet=$1
+
+ stop $facet
+ mount_facet $facet
+}
+
reboot_facet() {
facet=$1
if [ "$FAILURE_MODE" = HARD ]; then
testnum=$((nodenum % numloads))
start_client_load ${clients[nodenum]} ${CLIENT_LOADS[testnum]}
done
+ # bug 22169: wait the background threads to start
+ sleep 2
}
# only for remote client
local facet=$1
# Use default policy if $2 is not passed by caller.
- #define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2)
- # as we are in process of changing obd_timeout in different ways
- # let's set MAX longer than that
- local MAX=${2:-$(( TIMEOUT * 4 ))}
+ local MAX=${2:-$(max_recovery_time)}
local var_svc=${facet}_svc
local procfile="*.${!var_svc}.recovery_status"
return ${PIPESTATUS[0]}
}
+do_nodev() {
+ do_node --verbose "$@"
+}
+
single_local_node () {
[ "$1" = "$HOSTNAME" ]
}
if single_local_node $rnodes; then
if $verbose; then
- do_node --verbose $rnodes "$@"
+ do_nodev $rnodes "$@"
else
do_node $rnodes "$@"
fi
do_node $HOST "$@"
}
+do_nodesv() {
+ do_nodes --verbose "$@"
+}
+
add() {
local facet=$1
shift
do_facet mds$num "lctl set_param -n mdt.$MDT.identity_upcall \"NONE\""
fi
- do_facet mds$num "lctl set_param -n mdt/$MDT/identity_flush \"-1\""
+ do_facet mds$num "lctl set_param -n mdt/$MDT/identity_flush=-1"
if [ $old = "NONE" ]; then
return 1
is_mounted () {
local mntpt=$1
+ [ -z $mntpt ] && return 1
local mounted=$(mounted_lustre_filesystems)
echo $mounted' ' | grep -w -q $mntpt' '
}
+is_empty_dir() {
+ [ $(find $1 -maxdepth 1 -print | wc -l) = 1 ] && return 0
+ return 1
+}
+
+# empty lustre filesystem may have empty directories lost+found and .lustre
+is_empty_fs() {
+ [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \
+ -print | wc -l) = 1 ] || return 1
+ [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found && return 0
+ [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre && return 0
+ return 1
+}
+
check_and_setup_lustre() {
nfs_client_mode && return
check_and_setup_lustre
}
+# Get all of the server target devices from a given server node and type.
+get_mnt_devs() {
+ local node=$1
+ local type=$2
+ local obd_type
+ local devs
+ local dev
+
+ case $type in
+ mdt) obd_type="osd" ;;
+ ost) obd_type="obdfilter" ;; # needs to be fixed when OST also uses an OSD
+ *) echo "invalid server type" && return 1 ;;
+ esac
+
+ devs=$(do_node $node "lctl get_param -n $obd_type.*.mntdev")
+ for dev in $devs; do
+ case $dev in
+ *loop*) do_node $node "losetup $dev" | \
+ sed -e "s/.*(//" -e "s/).*//" ;;
+ *) echo $dev ;;
+ esac
+ done
+}
+
+# Get all of the server target devices.
+get_svr_devs() {
+ local i
+
+ # MDT device
+ MDTDEV=$(get_mnt_devs $(mdts_nodes) mdt)
+
+ # OST devices
+ i=0
+ for node in $(osts_nodes); do
+ OSTDEVS[i]=$(get_mnt_devs $node ost)
+ i=$((i + 1))
+ done
+}
+
+# Run e2fsck on MDT or OST device.
+run_e2fsck() {
+ local node=$1
+ local target_dev=$2
+ local ostidx=$3
+ local ostdb_opt=$4
+
+ df > /dev/null # update statfs data on disk
+ local cmd="$E2FSCK -d -v -f -n $MDSDB_OPT $ostdb_opt $target_dev"
+ echo $cmd
+ do_node $node $cmd
+ local rc=${PIPESTATUS[0]}
+ [ $rc -le $FSCK_MAX_ERR ] || \
+ error "$cmd returned $rc, should be <= $FSCK_MAX_ERR"
+ return 0
+}
+
+# Run e2fsck on MDT and OST(s) to generate databases used for lfsck.
+generate_db() {
+ local i
+ local ostidx
+ local dev
+ local tmp_file
+
+ [ $MDSCOUNT -eq 1 ] || error "CMD is not supported"
+ tmp_file=$(mktemp -p $SHARED_DIRECTORY ||
+ error "fail to create file in $SHARED_DIRECTORY")
+
+ # make sure everything gets to the backing store
+ local list=$(comma_list $CLIENTS $(facet_host $SINGLEMDS) $(osts_nodes))
+ do_nodes $list "sync; sleep 2; sync"
+
+ do_nodes $list ls $tmp_file || \
+ error "$SHARED_DIRECTORY is not a shared directory"
+ rm $tmp_file
+
+ run_e2fsck $(mdts_nodes) $MDTDEV
+
+ i=0
+ ostidx=0
+ OSTDB_LIST=""
+ for node in $(osts_nodes); do
+ for dev in ${OSTDEVS[i]}; do
+ local ostdb_opt=`eval echo $OSTDB_OPT`
+ run_e2fsck $node $dev $ostidx "$ostdb_opt"
+ OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx"
+ ostidx=$((ostidx + 1))
+ done
+ i=$((i + 1))
+ done
+}
+
+run_lfsck() {
+ local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
+ echo $cmd
+ eval $cmd
+ local rc=${PIPESTATUS[0]}
+ [ $rc -le $FSCK_MAX_ERR ] || \
+ error "$cmd returned $rc, should be <= $FSCK_MAX_ERR"
+ echo "lfsck finished with rc=$rc"
+
+ rm -rvf $MDSDB* $OSTDB* || true
+
+ return $rc
+}
+
check_and_cleanup_lustre() {
+ if [ "$LFSCK_ALWAYS" = "yes" ]; then
+ get_svr_devs
+ generate_db
+ if [ "$SKIP_LFSCK" == "no" ]; then
+ local rc=0
+ run_lfsck || rc=$?
+ else
+ echo "skip lfsck"
+ fi
+ fi
+
if is_mounted $MOUNT; then
[ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]*
[ "$ENABLE_QUOTA" ] && restore_quota_type || true
# print a newline if the last test was skipped
export LAST_SKIPPED=
+#
+# Main entry into test-framework. This is called with the name and
+# description of a test. The name is used to find the function to run
+# the test using "test_$name".
+#
+# This supports a variety of methods of specifying specific test to
+# run or not run. These need to be documented...
+#
run_test() {
assert_DIR
return $?
}
-EQUALS="======================================================================"
equals_msg() {
- msg="$@"
-
- local suffixlen=$((${#EQUALS} - ${#msg}))
- [ $suffixlen -lt 5 ] && suffixlen=5
- log `echo $(printf '===== %s %.*s\n' "$msg" $suffixlen $EQUALS)`
+ banner "$*"
}
log() {
echo done.
}
+
+#
+# Log a message (on all nodes) padded with "=" before and after.
+# Also appends a timestamp and prepends the testsuite name.
+#
+
+EQUALS="===================================================================================================="
+banner() {
+ msg="== ${TESTSUITE} $*"
+ last=${msg: -1:1}
+ [[ $last != "=" && $last != " " ]] && msg="$msg "
+ msg=$(printf '%s%.*s' "$msg" $((${#EQUALS} - ${#msg})) $EQUALS )
+ # always include at least == after the message
+ log "$msg== $(date +"%H:%M:%S (%s)")"
+}
+
+#
+# Run a single test function and cleanup after it.
+#
+# This function should be run in a subshell so the test func can
+# exit() without stopping the whole script.
+#
run_one() {
local testnum=$1
local message=$2
- local start_tm=$3
tfile=f${testnum}
export tdir=d0.${TESTSUITE}/d${base}
export TESTNAME=test_$testnum
local SAVE_UMASK=`umask`
umask 0022
- log "== test $testnum: $message == `date +%H:%M:%S` ($start_tm)"
+ banner "test $testnum: $message"
test_${testnum} || error "test_$testnum failed with $?"
cd $SAVE_PWD
reset_fail_loc
return 0
}
+#
+# Wrapper around run_one to ensure:
+# - test runs in subshell
+# - output of test is saved to separate log file for error reporting
+# - test result is saved to data file
+#
run_one_logged() {
local BEFORE=`date +%s`
local TEST_ERROR
rm -rf $LOGDIR/err
echo
- run_one $1 "$2" $BEFORE 2>&1 | tee $test_log
+ log_sub_test_begin test_${1}
+ (run_one $1 "$2") 2>&1 | tee $test_log
local RC=${PIPESTATUS[0]}
[ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \
duration=$((`date +%s` - $BEFORE))
pass "(${duration}s)"
[ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
- log_sub_test test_${1} $TEST_STATUS $duration "$RC" "$TEST_ERROR"
+ log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
if [ -f $LOGDIR/err ]; then
$FAIL_ON_ERROR && exit $RC
generate_machine_file() {
local nodes=${1//,/ }
local machinefile=$2
- rm -f $machinefile || error "can't rm $machinefile"
+ rm -f $machinefile
for node in $nodes; do
- echo $node >>$machinefile
+ echo $node >>$machinefile || \
+ { echo "can not generate machinefile $machinefile" && return 1; }
done
}
[ -z $nfsserver ] && echo "$dir is not nfs mounted" && return 1
- do_node --verbose $nfsserver lfs setstripe "$@"
+ do_nodev $nfsserver lfs setstripe "$@"
}
check_runas_id_ret() {
# generate a stream of formatted strings (<node> <param name>=<param value>)
save_lustre_params() {
local s
- do_nodes --verbose $1 "lctl get_param $2 | while read s; do echo \\\$s; done"
+ do_nodesv $1 "lctl get_param $2 | while read s; do echo \\\$s; done"
}
# restore lustre parameters from input stream, produces by save_lustre_params
# Add paths to lustre tests for 32 and 64 bit systems.
local RPATH="$RLUSTRE/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:$PATH"
- do_nodes --verbose $list "PATH=$RPATH sh rpc.sh $@ "
+ do_nodesv $list "PATH=$RPATH sh rpc.sh $@ "
}
wait_clients_import_state () {
return
fi
- do_nodes --verbose $list \
+ do_nodesv $list \
"$LCTL dk > ${prefix}.debug_log.\\\$(hostname).${suffix};
dmesg > ${prefix}.dmesg.\\\$(hostname).${suffix}"
if [ ! -f $LOGDIR/shared ]; then
return $rc
}
+# target_start_and_reset_recovery_timer()
+# service_time = at_est2timeout(service_time);
+# service_time += 2 * (CONNECTION_SWITCH_MAX + CONNECTION_SWITCH_INC +
+# INITIAL_CONNECT_TIMEOUT);
+# CONNECTION_SWITCH_MAX : min(25U, max(CONNECTION_SWITCH_MIN,obd_timeout))
+#define CONNECTION_SWITCH_INC 1
+#define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20)
+#define CONNECTION_SWITCH_MIN 5U
+
+max_recovery_time () {
+ local init_connect_timeout=$(( TIMEOUT / 20 ))
+ [[ $init_connect_timeout > 5 ]] || init_connect_timeout=5
+
+ local service_time=$(( $(at_max_get client) + $(( 2 * $(( 25 + 1 + init_connect_timeout)) )) ))
+
+ echo $service_time
+}
+
get_clients_mount_count () {
local clients=${CLIENTS:-`hostname`}
yml_log_test $1 >> $YAML_LOG
}
-log_sub_test() {
- yml_log_sub_test $@ >> $YAML_LOG
+log_test_status() {
+ yml_log_test_status $@ >> $YAML_LOG
+}
+
+log_sub_test_begin() {
+ yml_log_sub_test_begin $@ >> $YAML_LOG
+}
+
+log_sub_test_end() {
+ yml_log_sub_test_end $@ >> $YAML_LOG
+}
+
+run_llverdev()
+{
+ local dev=$1
+ local devname=$(basename $1)
+ local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}')
+ # loop devices aren't in /proc/partitions
+ [ "x$size" == "x" ] && local size=$(ls -l $dev | awk '{print $5}')
+
+ size=$(($size / 1024 / 1024)) # Gb
+
+ local partial_arg=""
+ # Run in partial (fast) mode if the size
+ # of a partition > 10 GB
+ [ $size -gt 10 ] && partial_arg="-p"
+
+ llverdev --force $partial_arg $dev
+}
+
+remove_mdt_files() {
+ local facet=$1
+ local mdtdev=$2
+ shift 2
+ local files="$@"
+ local mntpt=${MOUNT%/*}/$facet
+
+ echo "removing files from $mdtdev on $facet: $files"
+ mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
+ rc=0;
+ for f in $files; do
+ rm $mntpt/ROOT/$f || { rc=$?; break; }
+ done
+ umount -f $mntpt || return $?
+ return $rc
}
+duplicate_mdt_files() {
+ local facet=$1
+ local mdtdev=$2
+ shift 2
+ local files="$@"
+ local mntpt=${MOUNT%/*}/$facet
+
+ echo "duplicating files on $mdtdev on $facet: $files"
+ mkdir -p $mntpt || return $?
+ mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
+
+ do_umount() {
+ trap 0
+ popd > /dev/null
+ rm $tmp
+ umount -f $mntpt
+ }
+ trap do_umount EXIT
+
+ tmp=$(mktemp $TMP/setfattr.XXXXXXXXXX)
+ pushd $mntpt/ROOT > /dev/null || return $?
+ rc=0
+ for f in $files; do
+ touch $f.bad || return $?
+ getfattr -n trusted.lov $f | sed "s#$f#&.bad#" > $tmp
+ rc=${PIPESTATUS[0]}
+ [ $rc -eq 0 ] || return $rc
+ setfattr --restore $tmp || return $?
+ done
+ do_umount
+}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff