X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=a0b0dd882e8bf971984e874bf8190198f1a27ca6;hp=81a4b7b73b381a0e89ccde3723e2667a4283f57a;hb=d44b89dce32f4de91a8ef6c07d2340c9ff75773b;hpb=78f29d9d53e1861b0e11e2b2abfa649980679f97 diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 81a4b7b7..a0b0dd8 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1,11 +1,11 @@ #!/bin/bash # vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4: -trap 'print_summary && echo "test-framework exiting on error"' ERR +trap 'print_summary && touch $TF_FAIL && \ + echo "test-framework exiting on error"' ERR set -e #set -x - export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} export VERBOSE=false @@ -111,7 +111,22 @@ init_test_env() { export E2LABEL=${E2LABEL:-e2label} export DUMPE2FS=${DUMPE2FS:-dumpe2fs} export E2FSCK=${E2FSCK:-e2fsck} + export LFSCK_BIN=${LFSCK_BIN:-lfsck} + export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check filesystem after each test suit + export SKIP_LFSCK=${SKIP_LFSCK:-"yes"} # bug 13698, change to "no" when fixed + export SHARED_DIRECTORY=${SHARED_DIRECTORY:-"/tmp"} + export FSCK_MAX_ERR=4 # File system errors left uncorrected + if [ "$SKIP_LFSCK" == "no" ]; then + if [ ! -x `which $LFSCK_BIN` ]; then + log "$($E2FSCK -V)" + error_exit "$E2FSCK does not support lfsck" + fi + export MDSDB=${MDSDB:-$SHARED_DIRECTORY/mdsdb} + export OSTDB=${OSTDB:-$SHARED_DIRECTORY/ostdb} + export MDSDB_OPT="--mdsdb $MDSDB" + export OSTDB_OPT="--ostdb $OSTDB-\$ostidx" + fi #[ -d /r ] && export ROOT=${ROOT:-/r} export TMP=${TMP:-$ROOT/tmp} export TESTSUITELOG=${TMP}/${TESTSUITE}.log @@ -248,6 +263,8 @@ load_module() { shift BASE=`basename $module $EXT` + module_loaded ${BASE} && return + # If no module arguments were passed, get them from $MODOPTS_, else from # modprobe.conf if [ $# -eq 0 ]; then @@ -259,7 +276,7 @@ load_module() { # Nothing in $MODOPTS_; try modprobe.conf set -- $(grep "^options\\s*\<${module}\>" $MODPROBECONF) # Get rid of "options $module" - (($# > 0)) && shift 2 + (($# > 0)) && shift 2 # Ensure we have accept=all for lnet if [ $(basename $module) = lnet ]; then @@ -275,8 +292,6 @@ load_module() { [ $# -gt 0 ] && echo "${module} options: '$*'" - module_loaded ${BASE} && return - # Note that insmod will ignore anything in modprobe.conf, which is why we're # passing options on the command-line. if [ "$BASE" == "lnet_selftest" ] && \ @@ -301,12 +316,6 @@ load_modules_local() { echo "Using modprobe to load modules" return 0 fi - if [ "$HAVE_MODULES" = true ]; then - # we already loaded - echo "Modules already loaded" - return 0 - fi - HAVE_MODULES=true echo Loading modules from $LUSTRE load_module ../libcfs/libcfs/libcfs @@ -392,8 +401,6 @@ unload_modules() { fi fi - HAVE_MODULES=false - check_mem_leak || return 254 echo "modules unloaded." @@ -886,6 +893,13 @@ shutdown_facet() { fi } +remount_facet() { + local facet=$1 + + stop $facet + mount_facet $facet +} + reboot_facet() { facet=$1 if [ "$FAILURE_MODE" = HARD ]; then @@ -1137,10 +1151,7 @@ wait_recovery_complete () { local facet=$1 # Use default policy if $2 is not passed by caller. - #define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2) - # as we are in process of changing obd_timeout in different ways - # let's set MAX longer than that - local MAX=${2:-$(( TIMEOUT * 4 ))} + local MAX=${2:-$(max_recovery_time)} local var_svc=${facet}_svc local procfile="*.${!var_svc}.recovery_status" @@ -1754,7 +1765,7 @@ switch_identity() { do_facet mds$num "lctl set_param -n mdt.$MDT.identity_upcall \"NONE\"" fi - do_facet mds$num "lctl set_param -n mdt/$MDT/identity_flush \"-1\"" + do_facet mds$num "lctl set_param -n mdt/$MDT/identity_flush=-1" if [ $old = "NONE" ]; then return 1 @@ -2046,11 +2057,26 @@ check_timeout () { is_mounted () { local mntpt=$1 + [ -z $mntpt ] && return 1 local mounted=$(mounted_lustre_filesystems) echo $mounted' ' | grep -w -q $mntpt' ' } +is_empty_dir() { + [ $(find $1 -maxdepth 1 -print | wc -l) = 1 ] && return 0 + return 1 +} + +# empty lustre filesystem may have empty directories lost+found and .lustre +is_empty_fs() { + [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \ + -print | wc -l) = 1 ] || return 1 + [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found && return 0 + [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre && return 0 + return 1 +} + check_and_setup_lustre() { nfs_client_mode && return @@ -2145,7 +2171,123 @@ cleanup_and_setup_lustre() { check_and_setup_lustre } +# Get all of the server target devices from a given server node and type. +get_mnt_devs() { + local node=$1 + local type=$2 + local obd_type + local devs + local dev + + case $type in + mdt) obd_type="osd" ;; + ost) obd_type="obdfilter" ;; # needs to be fixed when OST also uses an OSD + *) echo "invalid server type" && return 1 ;; + esac + + devs=$(do_node $node "lctl get_param -n $obd_type.*.mntdev") + for dev in $devs; do + case $dev in + *loop*) do_node $node "losetup $dev" | \ + sed -e "s/.*(//" -e "s/).*//" ;; + *) echo $dev ;; + esac + done +} + +# Get all of the server target devices. +get_svr_devs() { + local i + + # MDT device + MDTDEV=$(get_mnt_devs $(mdts_nodes) mdt) + + # OST devices + i=0 + for node in $(osts_nodes); do + OSTDEVS[i]=$(get_mnt_devs $node ost) + i=$((i + 1)) + done +} + +# Run e2fsck on MDT or OST device. +run_e2fsck() { + local node=$1 + local target_dev=$2 + local ostidx=$3 + local ostdb_opt=$4 + + df > /dev/null # update statfs data on disk + local cmd="$E2FSCK -d -v -f -n $MDSDB_OPT $ostdb_opt $target_dev" + echo $cmd + do_node $node $cmd + local rc=${PIPESTATUS[0]} + [ $rc -le $FSCK_MAX_ERR ] || \ + error "$cmd returned $rc, should be <= $FSCK_MAX_ERR" + return 0 +} + +# Run e2fsck on MDT and OST(s) to generate databases used for lfsck. +generate_db() { + local i + local ostidx + local dev + local tmp_file + + [ $MDSCOUNT -eq 1 ] || error "CMD is not supported" + tmp_file=$(mktemp -p $SHARED_DIRECTORY || + error "fail to create file in $SHARED_DIRECTORY") + + # make sure everything gets to the backing store + local list=$(comma_list $CLIENTS $(facet_host $SINGLEMDS) $(osts_nodes)) + do_nodes $list "sync; sleep 2; sync" + + do_nodes $list ls $tmp_file || \ + error "$SHARED_DIRECTORY is not a shared directory" + rm $tmp_file + + run_e2fsck $(mdts_nodes) $MDTDEV + + i=0 + ostidx=0 + OSTDB_LIST="" + for node in $(osts_nodes); do + for dev in ${OSTDEVS[i]}; do + local ostdb_opt=`eval echo $OSTDB_OPT` + run_e2fsck $node $dev $ostidx "$ostdb_opt" + OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx" + ostidx=$((ostidx + 1)) + done + i=$((i + 1)) + done +} + +run_lfsck() { + local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT" + echo $cmd + eval $cmd + local rc=${PIPESTATUS[0]} + [ $rc -le $FSCK_MAX_ERR ] || \ + error "$cmd returned $rc, should be <= $FSCK_MAX_ERR" + echo "lfsck finished with rc=$rc" + + rm -rvf $MDSDB* $OSTDB* || true + + return $rc +} + check_and_cleanup_lustre() { + if [ "$LFSCK_ALWAYS" = "yes" ]; then + get_svr_devs + generate_db + if [ "$SKIP_LFSCK" == "no" ]; then + local rc=0 + run_lfsck || rc=$? + else + echo "skip lfsck" + fi + fi + if is_mounted $MOUNT; then [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* [ "$ENABLE_QUOTA" ] && restore_quota_type || true @@ -2716,6 +2858,7 @@ run_one_logged() { rm -rf $LOGDIR/err echo + log_sub_test_begin test_${1} (run_one $1 "$2") 2>&1 | tee $test_log local RC=${PIPESTATUS[0]} @@ -2725,7 +2868,7 @@ run_one_logged() { duration=$((`date +%s` - $BEFORE)) pass "(${duration}s)" [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err) - log_sub_test test_${1} $TEST_STATUS $duration "$RC" "$TEST_ERROR" + log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR" if [ -f $LOGDIR/err ]; then $FAIL_ON_ERROR && exit $RC @@ -3611,7 +3754,7 @@ combination() R=0 else N=$((N + 1)) - while [ $N -lt $M ]; do + while [ $N -le $M ]; do R=$((R * N)) N=$((N + 1)) done @@ -3822,7 +3965,6 @@ wait_flavor() return 0 else echo "found $res $flavor connections of $dir, not ready ($expect)" - return 0 sleep 4 fi done @@ -3932,8 +4074,16 @@ log_test() { yml_log_test $1 >> $YAML_LOG } -log_sub_test() { - yml_log_sub_test $@ >> $YAML_LOG +log_test_status() { + yml_log_test_status $@ >> $YAML_LOG +} + +log_sub_test_begin() { + yml_log_sub_test_begin $@ >> $YAML_LOG +} + +log_sub_test_end() { + yml_log_sub_test_end $@ >> $YAML_LOG } run_llverdev() @@ -3941,6 +4091,9 @@ run_llverdev() local dev=$1 local devname=$(basename $1) local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}') + # loop devices aren't in /proc/partitions + [ "x$size" == "x" ] && local size=$(ls -l $dev | awk '{print $5}') + size=$(($size / 1024 / 1024)) # Gb local partial_arg="" @@ -3950,3 +4103,52 @@ run_llverdev() llverdev --force $partial_arg $dev } + +remove_mdt_files() { + local facet=$1 + local mdtdev=$2 + shift 2 + local files="$@" + local mntpt=${MOUNT%/*}/$facet + + echo "removing files from $mdtdev on $facet: $files" + mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $? + rc=0; + for f in $files; do + rm $mntpt/ROOT/$f || { rc=$?; break; } + done + umount -f $mntpt || return $? + return $rc +} + +duplicate_mdt_files() { + local facet=$1 + local mdtdev=$2 + shift 2 + local files="$@" + local mntpt=${MOUNT%/*}/$facet + + echo "duplicating files on $mdtdev on $facet: $files" + mkdir -p $mntpt || return $? + mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $? + + do_umount() { + trap 0 + popd > /dev/null + rm $tmp + umount -f $mntpt + } + trap do_umount EXIT + + tmp=$(mktemp $TMP/setfattr.XXXXXXXXXX) + pushd $mntpt/ROOT > /dev/null || return $? + rc=0 + for f in $files; do + touch $f.bad || return $? + getfattr -n trusted.lov $f | sed "s#$f#&.bad#" > $tmp + rc=${PIPESTATUS[0]} + [ $rc -eq 0 ] || return $rc + setfattr --restore $tmp || return $? + done + do_umount +}