X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=24031dd5a370f8e306fb965e0a4c210e8c80768d;hp=be4ae464e30f4fc205841af63f15ead388850759;hb=1b8ff9af55071e574ff7b875bdb03294e79da3d2;hpb=02b8c957baa3b8742bbcde895a22ad498c6fbe25 diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index be4ae46..24031dd 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -15,6 +15,7 @@ export GSS=false export GSS_KRB5=false export GSS_PIPEFS=false export IDENTITY_UPCALL=default +export QUOTA_AUTO=1 #export PDSH="pdsh -S -Rssh -w" @@ -22,6 +23,16 @@ export IDENTITY_UPCALL=default LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/functions.sh +LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg} + +EXCEPT_LIST_FILE=${EXCEPT_LIST_FILE:-${LUSTRE_TESTS_CFG_DIR}/tests-to-skip.sh} + +if [ -f "$EXCEPT_LIST_FILE" ]; then + echo "Reading test skip list from $EXCEPT_LIST_FILE" + cat $EXCEPT_LIST_FILE + . $EXCEPT_LIST_FILE +fi + assert_DIR () { local failed="" [[ $DIR/ = $MOUNT/* ]] || \ @@ -204,13 +215,20 @@ case `uname -r` in *) EXT=".ko"; USE_QUOTA=yes;; esac + +module_loaded () { + /sbin/lsmod | grep -q $1 +} + load_module() { EXT=".ko" module=$1 shift BASE=`basename $module $EXT` - lsmod | grep -q ${BASE} || \ - if [ -f ${LUSTRE}/${module}${EXT} ]; then + + module_loaded ${BASE} && return + + if [ -f ${LUSTRE}/${module}${EXT} ]; then insmod ${LUSTRE}/${module}${EXT} $@ else # must be testing a "make install" or "rpm" installation @@ -538,9 +556,7 @@ restore_quota_type () { setup_quota(){ local mntpt=$1 - # We need: - # 1. run quotacheck only if quota is off - # 2. save the original quota_type params, restore them after testing + # We need save the original quota_type params, and restore them after testing # Suppose that quota type the same on mds and ost local quota_type=$(quota_type | grep MDT | cut -d "=" -f2) @@ -549,6 +565,9 @@ setup_quota(){ if [ "$quota_type" != "$QUOTA_TYPE" ]; then export old_QUOTA_TYPE=$quota_type quota_save_version $QUOTA_TYPE + else + qtype=$(tr -c -d "ug" <<< $QUOTA_TYPE) + $LFS quotacheck -$qtype $mntpt || error "quotacheck has failed for $type" fi local quota_usrs=$QUOTA_USERS @@ -691,14 +710,13 @@ sanity_mount_check () { # mount clients if not mouted zconf_mount_clients() { - local OPTIONS local clients=$1 local mnt=$2 - + local OPTIONS=${3:-$MOUNTOPT} # Only supply -o to mount if we have options - if [ -n "$MOUNTOPT" ]; then - OPTIONS="-o $MOUNTOPT" + if [ "$OPTIONS" ]; then + OPTIONS="-o $OPTIONS" fi local device=$MGSNID:/$FSNAME if [ -z "$mnt" -o -z "$FSNAME" ]; then @@ -738,7 +756,7 @@ zconf_umount_clients() { echo "Stopping clients: $clients $mnt (opts:$force)" do_nodes $clients "running=\\\$(grep -c $mnt' ' /proc/mounts); if [ \\\$running -ne 0 ] ; then -echo Stopping client \\\$(hostname) client $mnt opts:$force; +echo Stopping client \\\$(hostname) $mnt opts:$force; lsof -t $mnt || need_kill=no; if [ "x$force" != "x" -a "x\\\$need_kill" != "xno" ]; then pids=\\\$(lsof -t $mnt | sort -u); @@ -966,7 +984,7 @@ cleanup_check() { [ "`lctl dl 2> /dev/null | wc -l`" -gt 0 ] && lctl dl && \ echo "$0: lustre didn't clean up..." 1>&2 && return 202 || true - if [ "`/sbin/lsmod 2>&1 | egrep 'lnet|libcfs'`" ]; then + if module_loaded lnet || module_loaded libcfs; then echo "$0: modules still loaded..." 1>&2 /sbin/lsmod 1>&2 return 203 @@ -1013,12 +1031,13 @@ wait_delete_completed () { sleep 1 TOTAL=`lctl get_param -n osc.*.kbytesavail | \ awk 'BEGIN{total=0}; {total+=$1}; END{print total}'` - [ "$TOTAL" -eq "$TOTALPREV" ] && break + [ "$TOTAL" -eq "$TOTALPREV" ] && return 0 echo "Waiting delete completed ... prev: $TOTALPREV current: $TOTAL " TOTALPREV=$TOTAL WAIT=$(( WAIT + 1)) done - echo "Delete completed." + echo "Delete is not completed in $MAX_WAIT sec" + return 1 } wait_for_host() { @@ -1036,12 +1055,12 @@ wait_for() { wait_recovery_complete () { local facet=$1 - # Use default policy if $2 is not passed by caller. + # Use default policy if $2 is not passed by caller. #define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2) # as we are in process of changing obd_timeout in different ways # let's set MAX longer than that local MAX=${2:-$(( TIMEOUT * 4 ))} - + local var_svc=${facet}_svc local procfile="*.${!var_svc}.recovery_status" local WAIT=0 @@ -1058,6 +1077,57 @@ wait_recovery_complete () { return 1 } +wait_mds_ost_sync () { + # just because recovery is done doesn't mean we've finished + # orphan cleanup. Wait for llogs to get synchronized. + echo "Waiting for orphan cleanup..." + # MAX value includes time needed for MDS-OST reconnection + local MAX=$(( TIMEOUT * 2 )) + local WAIT=0 + while [ $WAIT -lt $MAX ]; do + local -a sync=($(do_nodes $(comma_list $(osts_nodes)) \ + "$LCTL get_param -n obdfilter.*.mds_sync")) + local con=1 + for ((i=0; i<${#sync[@]}; i++)); do + [ ${sync[$i]} -eq 0 ] && continue + # there is a not finished MDS-OST synchronization + con=0 + break; + done + sleep 2 # increase waiting time and cover statfs cache + [ ${con} -eq 1 ] && return 0 + echo "Waiting $WAIT secs for $facet mds-ost sync done." + WAIT=$((WAIT + 2)) + done + echo "$facet recovery not done in $MAX sec. $STATUS" + return 1 +} + +wait_destroy_complete () { + echo "Waiting for destroy to be done..." + # MAX value shouldn't be big as this mean server responsiveness + # never increase this just to make test pass but investigate + # why it takes so long time + local MAX=5 + local WAIT=0 + while [ $WAIT -lt $MAX ]; do + local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight)) + local con=1 + for ((i=0; i<${#RPCs[@]}; i++)); do + [ ${RPCs[$i]} -eq 0 ] && continue + # there are still some destroy RPCs in flight + con=0 + break; + done + sleep 1 + [ ${con} -eq 1 ] && return 0 # done waiting + echo "Waiting $WAIT secs for destroys to be done." + WAIT=$((WAIT + 1)) + done + echo "Destroys weren't done in $MAX sec." + return 1 +} + wait_exit_ST () { local facet=$1 @@ -1111,15 +1181,31 @@ wait_remote_prog () { return $rc } -client_df() { +clients_up() { # not every config has many clients + sleep 1 if [ ! -z "$CLIENTS" ]; then - $PDSH $CLIENTS "df $MOUNT" > /dev/null + $PDSH $CLIENTS "stat -f $MOUNT" > /dev/null else - df $MOUNT > /dev/null + stat -f $MOUNT > /dev/null fi } +client_up() { + local client=$1 + # usually checked on particular client or locally + sleep 1 + if [ ! -z "$client" ]; then + $PDSH $client "stat -f $MOUNT" > /dev/null + else + stat -f $MOUNT > /dev/null + fi +} + +client_evicted() { + ! client_up $1 +} + client_reconnect() { uname -n >> $MOUNT/recon if [ -z "$CLIENTS" ]; then @@ -1195,7 +1281,7 @@ ost_evict_client() { fail() { facet_failover $* || error "failover: $?" - client_df || error "post-failover df: $?" + clients_up || error "post-failover df: $?" } fail_nodf() { @@ -1208,9 +1294,8 @@ fail_abort() { stop $facet change_active $facet mount_facet $facet -o abort_recovery - client_df || echo "first df failed: $?" - sleep 1 - client_df || error "post-failover df: $?" + clients_up || echo "first df failed: $?" + clients_up || error "post-failover df: $?" } do_lmc() { @@ -1767,10 +1852,19 @@ init_param_vars () { if [ x"$(som_check)" = x"enabled" ]; then ENABLE_QUOTA="" + echo "disable quota temporary when SOM enabled" fi - if [ "$ENABLE_QUOTA" ]; then - setup_quota $MOUNT || return 2 + if [ $QUOTA_AUTO -ne 0 ]; then + if [ "$ENABLE_QUOTA" ]; then + echo "enable quota as required" + setup_quota $MOUNT || return 2 + else + echo "disable quota as required" + $LFS quotaoff -ug $MOUNT > /dev/null 2>&1 + fi fi + + return 0 } nfs_client_mode () { @@ -1788,9 +1882,7 @@ nfs_client_mode () { return 1 } -check_config () { - nfs_client_mode && return - +check_config_client () { local mntpt=$1 local mounted=$(mount | grep " $mntpt ") @@ -1809,8 +1901,8 @@ check_config () { if [[ x$mgc != xMGC$MGSNID ]]; then if [ "$mgs_HOST" ]; then local mgc_ip=$(ping -q -c1 -w1 $mgs_HOST | grep PING | awk '{print $3}' | sed -e "s/(//g" -e "s/)//g") - [[ x$mgc = xMGC$mgc_ip@$NETTYPE ]] || - error_exit "MGSNID=$MGSNID, mounted: $mounted, MGC : $mgc" +# [[ x$mgc = xMGC$mgc_ip@$NETTYPE ]] || +# error_exit "MGSNID=$MGSNID, mounted: $mounted, MGC : $mgc" fi fi return 0 @@ -1825,10 +1917,20 @@ check_config () { local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}') mgshost=$(echo $mgshost | awk -F: '{print $1}') -# if [ "$mgshost" != "$myMGS_host" ]; then -# error_exit "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE -# Please use correct config or set mds_HOST correctly!" -# fi + if [ "$mgshost" != "$myMGS_host" ]; then + log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE + Please use correct config or set mds_HOST correctly!" + fi + +} + +check_config_clients () { + local clients=${CLIENTS:-$HOSTNAME} + local mntpt=$1 + + nfs_client_mode && return + + do_rpc_nodes $clients check_config_client $mntpt sanity_mount_check || error "environments are insane!" @@ -1843,37 +1945,65 @@ check_timeout () { fi } +is_mounted () { + local mntpt=$1 + local mounted=$(mounted_lustre_filesystems) + + echo $mounted' ' | grep -w -q $mntpt' ' +} + check_and_setup_lustre() { nfs_client_mode && return local MOUNTED=$(mounted_lustre_filesystems) local do_check=true - # MOUNT is not mounted - if [ -z "$MOUNTED" ] || ! $(echo $MOUNTED | grep -w -q $MOUNT); then + # 1. + # both MOUNT and MOUNT2 are not mounted + if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then [ "$REFORMAT" ] && formatall + # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set) setupall - MOUNTED=$(mounted_lustre_filesystems | head -1) - [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted" + is_mounted $MOUNT || error "NAME=$NAME not mounted" export I_MOUNTED=yes do_check=false - - # MOUNT and MOUNT2 are mounted - elif $(echo $MOUNTED | grep -w -q $MOUNT2); then - - # MOUNT2 is mounted, MOUNT_2 is not set - if ! [ "$MOUNT_2" ]; then - zconf_umount `hostname` $MOUNT2 - export I_UMOUNTED2=yes - - # MOUNT2 is mounted, MOUNT_2 is set - else - check_config $MOUNT2 - fi + # 2. + # MOUNT2 is mounted + elif is_mounted $MOUNT2; then + # 3. + # MOUNT2 is mounted, while MOUNT_2 is not set + if ! [ "$MOUNT_2" ]; then + cleanup_mount $MOUNT2 + export I_UMOUNTED2=yes + + # 4. + # MOUNT2 is mounted, MOUNT_2 is set + else + # FIXME: what to do if check_config failed? + # i.e. if: + # 1) remote client has mounted other Lustre fs ? + # 2) it has insane env ? + # let's try umount MOUNT2 on all clients and mount it again: + if ! check_config_clients $MOUNT2; then + cleanup_mount $MOUNT2 + restore_mount $MOUNT2 + export I_MOUNTED2=yes + fi + fi + + # 5. + # MOUNT is mounted MOUNT2 is not mounted + elif [ "$MOUNT_2" ]; then + restore_mount $MOUNT2 + export I_MOUNTED2=yes fi if $do_check; then - check_config $MOUNT + # FIXME: what to do if check_config failed? + # i.e. if: + # 1) remote client has mounted other Lustre fs? + # 2) lustre is mounted on remote_clients atall ? + check_config_clients $MOUNT init_facets_vars init_param_vars @@ -1887,6 +2017,20 @@ check_and_setup_lustre() { fi } +restore_mount () { + local clients=${CLIENTS:-$HOSTNAME} + local mntpt=$1 + + zconf_mount_clients $clients $mntpt +} + +cleanup_mount () { + local clients=${CLIENTS:-$HOSTNAME} + local mntpt=$1 + + zconf_umount_clients $clients $mntpt +} + cleanup_and_setup_lustre() { if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then lctl set_param debug=0 || true @@ -1899,18 +2043,23 @@ cleanup_and_setup_lustre() { } check_and_cleanup_lustre() { - if [ "`mount | grep $MOUNT`" ]; then + if is_mounted $MOUNT; then [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* [ "$ENABLE_QUOTA" ] && restore_quota_type || true fi + if [ "$I_UMOUNTED2" = "yes" ]; then - mount_client $MOUNT2 || error "restore $MOUNT2 failed" + restore_mount $MOUNT2 || error "restore $MOUNT2 failed" + fi + + if [ "$I_MOUNTED2" = "yes" ]; then + cleanup_mount $MOUNT2 fi if [ "$I_MOUNTED" = "yes" ]; then cleanupall -f || error "cleanup failed" + unset I_MOUNTED fi - unset I_MOUNTED } ####### @@ -1982,9 +2131,9 @@ testslist_filter () { local start_at=$START_AT local stop_at=$STOP_AT - local var=${TESTSUITE}_START_AT + local var=${TESTSUITE//-/_}_START_AT [ x"${!var}" != x ] && start_at=${!var} - var=${TESTSUITE}_STOP_AT + var=${TESTSUITE//-/_}_STOP_AT [ x"${!var}" != x ] && stop_at=${!var} sed -n 's/^test_\([^ (]*\).*/\1/p' $script | \ @@ -2219,7 +2368,6 @@ stop_full_debug_logging() { error_noexit() { local TYPE=${TYPE:-"FAIL"} local ERRLOG - lctl set_param fail_loc=0 2>/dev/null || true local dump=true # do not dump logs if $1=false @@ -2243,7 +2391,10 @@ error_noexit() { error() { error_noexit "$@" - $FAIL_ON_ERROR && exit 1 || true + if $FAIL_ON_ERROR; then + reset_fail_loc + exit 1 + fi } error_exit() { @@ -2366,7 +2517,7 @@ equals_msg() { log() { echo "$*" - lsmod | grep lnet > /dev/null || load_modules + module_loaded lnet || load_modules local MSG="$*" # Get rid of ' @@ -2869,12 +3020,13 @@ get_mds_dir () { local dir=$1 local file=$dir/f0.get_mds_dir_tmpfile + mkdir -p $dir rm -f $file sleep 1 local iused=$(lfs df -i $dir | grep MDT | awk '{print $3}') local -a oldused=($iused) - touch $file + openfile -f O_CREAT:O_LOV_DELAY_CREATE -m 0644 $file > /dev/null sleep 1 iused=$(lfs df -i $dir | grep MDT | awk '{print $3}') local -a newused=($iused) @@ -3049,7 +3201,6 @@ wait_clients_import_state () { *) error "unknown facet!" ;; esac - if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then error "import is not in ${expected} state" return 1 @@ -3073,6 +3224,102 @@ oos_full() { return $OSCFULL } +pool_list () { + do_facet mgs lctl pool_list $1 +} + +create_pool() { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + do_facet mgs lctl pool_new $1 + local RC=$? + # get param should return err unless pool is created + [[ $RC -ne 0 ]] && return $RC + + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "" || RC=1 + if [[ $RC -eq 0 ]]; then + add_pool_to_list $1 + else + error "pool_new failed $1" + fi + return $RC +} + +add_pool_to_list () { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + local listvar=${fsname}_CREATED_POOLS + eval export ${listvar}=$(expand_list ${!listvar} $poolname) +} + +remove_pool_from_list () { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + local listvar=${fsname}_CREATED_POOLS + eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname) +} + +destroy_pool_int() { + local ost + local OSTS=$(do_facet $SINGLEMDS lctl pool_list $1 | \ + awk '$1 !~ /^Pool:/ {print $1}') + for ost in $OSTS; do + do_facet mgs lctl pool_remove $1 $ost + done + do_facet mgs lctl pool_destroy $1 +} + +# . or +destroy_pool() { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + [[ x$fsname = x$poolname ]] && fsname=$FSNAME + + local RC + + pool_list $fsname.$poolname || return $? + + destroy_pool_int $fsname.$poolname + RC=$? + [[ $RC -ne 0 ]] && return $RC + + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || RC=1 + + if [[ $RC -eq 0 ]]; then + remove_pool_from_list $fsname.$poolname + else + error "destroy pool failed $1" + fi + return $RC +} + +destroy_pools () { + local fsname=${1:-$FSNAME} + local poolname + local listvar=${fsname}_CREATED_POOLS + + pool_list $fsname + + [ x${!listvar} = x ] && return 0 + + echo destroy the created pools: ${!listvar} + for poolname in ${!listvar//,/ }; do + destroy_pool $fsname.$poolname + done +} + +cleanup_pools () { + local fsname=${1:-$FSNAME} + trap 0 + destroy_pools $fsname +} + gather_logs () { local list=$1 @@ -3114,3 +3361,26 @@ cleanup_logs () { [ -n ${TESTSUITE} ] && do_nodes $list "rm -f $TMP/*${TESTSUITE}*" || true } +do_ls () { + local mntpt_root=$1 + local num_mntpts=$2 + local dir=$3 + local i + local cmd + local pids + local rc=0 + + for i in $(seq 0 $num_mntpts); do + cmd="ls -laf ${mntpt_root}$i/$dir" + echo + $cmd; + $cmd > /dev/null & + pids="$pids $!" + done + echo pids=$pids + for pid in $pids; do + wait $pid || rc=$? + done + + return $rc +} +