X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=919d90f7dd14dee3b999f81cea8d58029f5f5522;hb=f78c9f5c45c52b0e5d6046a9dbe2a27ee62d9594;hp=1da55a3304c3030be601e269c9cad305d14d14c2;hpb=c431c0df2ae663f977abd4433f44b90120ca6f77;p=fs%2Flustre-release.git diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 1da55a3..919d90f 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -23,6 +23,16 @@ export QUOTA_AUTO=1 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/functions.sh +LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg} + +EXCEPT_LIST_FILE=${EXCEPT_LIST_FILE:-${LUSTRE_TESTS_CFG_DIR}/tests-to-skip.sh} + +if [ -f "$EXCEPT_LIST_FILE" ]; then + echo "Reading test skip list from $EXCEPT_LIST_FILE" + cat $EXCEPT_LIST_FILE + . $EXCEPT_LIST_FILE +fi + assert_DIR () { local failed="" [[ $DIR/ = $MOUNT/* ]] || \ @@ -1171,15 +1181,31 @@ wait_remote_prog () { return $rc } -client_df() { +clients_up() { # not every config has many clients + sleep 1 if [ ! -z "$CLIENTS" ]; then - $PDSH $CLIENTS "df $MOUNT" > /dev/null + $PDSH $CLIENTS "stat -f $MOUNT" > /dev/null + else + stat -f $MOUNT > /dev/null + fi +} + +client_up() { + local client=$1 + # usually checked on particular client or locally + sleep 1 + if [ ! -z "$client" ]; then + $PDSH $client "stat -f $MOUNT" > /dev/null else - df $MOUNT > /dev/null + stat -f $MOUNT > /dev/null fi } +client_evicted() { + ! client_up $1 +} + client_reconnect() { uname -n >> $MOUNT/recon if [ -z "$CLIENTS" ]; then @@ -1255,7 +1281,7 @@ ost_evict_client() { fail() { facet_failover $* || error "failover: $?" - client_df || error "post-failover df: $?" + clients_up || error "post-failover df: $?" } fail_nodf() { @@ -1268,9 +1294,8 @@ fail_abort() { stop $facet change_active $facet mount_facet $facet -o abort_recovery - client_df || echo "first df failed: $?" - sleep 1 - client_df || error "post-failover df: $?" + clients_up || echo "first df failed: $?" + clients_up || error "post-failover df: $?" } do_lmc() { @@ -1838,6 +1863,8 @@ init_param_vars () { $LFS quotaoff -ug $MOUNT > /dev/null 2>&1 fi fi + + return 0 } nfs_client_mode () { @@ -1855,9 +1882,7 @@ nfs_client_mode () { return 1 } -check_config () { - nfs_client_mode && return - +check_config_client () { local mntpt=$1 local mounted=$(mount | grep " $mntpt ") @@ -1892,10 +1917,20 @@ check_config () { local mgshost=$(mount | grep " $mntpt " | awk -F@ '{print $1}') mgshost=$(echo $mgshost | awk -F: '{print $1}') - if [ "$mgshost" != "$myMGS_host" ]; then - log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE - Please use correct config or set mds_HOST correctly!" - fi +# if [ "$mgshost" != "$myMGS_host" ]; then +# log "Bad config file: lustre is mounted with mgs $mgshost, but mgs_HOST=$mgs_HOST, NETTYPE=$NETTYPE +# Please use correct config or set mds_HOST correctly!" +# fi + +} + +check_config_clients () { + local clients=${CLIENTS:-$HOSTNAME} + local mntpt=$1 + + nfs_client_mode && return + + do_rpc_nodes $clients check_config_client $mntpt sanity_mount_check || error "environments are insane!" @@ -1910,37 +1945,65 @@ check_timeout () { fi } +is_mounted () { + local mntpt=$1 + local mounted=$(mounted_lustre_filesystems) + + echo $mounted' ' | grep -w -q $mntpt' ' +} + check_and_setup_lustre() { nfs_client_mode && return local MOUNTED=$(mounted_lustre_filesystems) local do_check=true - # MOUNT is not mounted - if [ -z "$MOUNTED" ] || ! $(echo $MOUNTED | grep -w -q $MOUNT); then + # 1. + # both MOUNT and MOUNT2 are not mounted + if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then [ "$REFORMAT" ] && formatall + # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set) setupall - MOUNTED=$(mounted_lustre_filesystems | head -1) - [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted" + is_mounted $MOUNT || error "NAME=$NAME not mounted" export I_MOUNTED=yes do_check=false - - # MOUNT and MOUNT2 are mounted - elif $(echo $MOUNTED | grep -w -q $MOUNT2); then - - # MOUNT2 is mounted, MOUNT_2 is not set - if ! [ "$MOUNT_2" ]; then - zconf_umount `hostname` $MOUNT2 - export I_UMOUNTED2=yes - - # MOUNT2 is mounted, MOUNT_2 is set - else - check_config $MOUNT2 - fi + # 2. + # MOUNT2 is mounted + elif is_mounted $MOUNT2; then + # 3. + # MOUNT2 is mounted, while MOUNT_2 is not set + if ! [ "$MOUNT_2" ]; then + cleanup_mount $MOUNT2 + export I_UMOUNTED2=yes + + # 4. + # MOUNT2 is mounted, MOUNT_2 is set + else + # FIXME: what to do if check_config failed? + # i.e. if: + # 1) remote client has mounted other Lustre fs ? + # 2) it has insane env ? + # let's try umount MOUNT2 on all clients and mount it again: + if ! check_config_clients $MOUNT2; then + cleanup_mount $MOUNT2 + restore_mount $MOUNT2 + export I_MOUNTED2=yes + fi + fi + + # 5. + # MOUNT is mounted MOUNT2 is not mounted + elif [ "$MOUNT_2" ]; then + restore_mount $MOUNT2 + export I_MOUNTED2=yes fi if $do_check; then - check_config $MOUNT + # FIXME: what to do if check_config failed? + # i.e. if: + # 1) remote client has mounted other Lustre fs? + # 2) lustre is mounted on remote_clients atall ? + check_config_clients $MOUNT init_facets_vars init_param_vars @@ -1954,6 +2017,20 @@ check_and_setup_lustre() { fi } +restore_mount () { + local clients=${CLIENTS:-$HOSTNAME} + local mntpt=$1 + + zconf_mount_clients $clients $mntpt +} + +cleanup_mount () { + local clients=${CLIENTS:-$HOSTNAME} + local mntpt=$1 + + zconf_umount_clients $clients $mntpt +} + cleanup_and_setup_lustre() { if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then lctl set_param debug=0 || true @@ -1966,18 +2043,23 @@ cleanup_and_setup_lustre() { } check_and_cleanup_lustre() { - if [ "`mount | grep $MOUNT`" ]; then + if is_mounted $MOUNT; then [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* [ "$ENABLE_QUOTA" ] && restore_quota_type || true fi + if [ "$I_UMOUNTED2" = "yes" ]; then - mount_client $MOUNT2 || error "restore $MOUNT2 failed" + restore_mount $MOUNT2 || error "restore $MOUNT2 failed" + fi + + if [ "$I_MOUNTED2" = "yes" ]; then + cleanup_mount $MOUNT2 fi if [ "$I_MOUNTED" = "yes" ]; then cleanupall -f || error "cleanup failed" + unset I_MOUNTED fi - unset I_MOUNTED } ####### @@ -2286,7 +2368,6 @@ stop_full_debug_logging() { error_noexit() { local TYPE=${TYPE:-"FAIL"} local ERRLOG - lctl set_param fail_loc=0 2>/dev/null || true local dump=true # do not dump logs if $1=false @@ -2310,7 +2391,10 @@ error_noexit() { error() { error_noexit "$@" - $FAIL_ON_ERROR && exit 1 || true + if $FAIL_ON_ERROR; then + reset_fail_loc + exit 1 + fi } error_exit() { @@ -3117,7 +3201,6 @@ wait_clients_import_state () { *) error "unknown facet!" ;; esac - if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then error "import is not in ${expected} state" return 1 @@ -3141,6 +3224,44 @@ oos_full() { return $OSCFULL } +pool_list () { + do_facet mgs lctl pool_list $1 +} + +create_pool() { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + do_facet mgs lctl pool_new $1 + local RC=$? + # get param should return err unless pool is created + [[ $RC -ne 0 ]] && return $RC + + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "" || RC=1 + if [[ $RC -eq 0 ]]; then + add_pool_to_list $1 + else + error "pool_new failed $1" + fi + return $RC +} + +add_pool_to_list () { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + local listvar=${fsname}_CREATED_POOLS + eval export ${listvar}=$(expand_list ${!listvar} $poolname) +} + +remove_pool_from_list () { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + local listvar=${fsname}_CREATED_POOLS + eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname) +} destroy_pool_int() { local ost @@ -3152,19 +3273,51 @@ destroy_pool_int() { do_facet mgs lctl pool_destroy $1 } +# . or destroy_pool() { + local fsname=${1%%.*} + local poolname=${1##$fsname.} + + [[ x$fsname = x$poolname ]] && fsname=$FSNAME + local RC - do_facet $SINGLEMDS lctl pool_list $FSNAME.$1 - RC=$? - [[ $RC -ne 0 ]] && return $RC + pool_list $fsname.$poolname || return $? - destroy_pool_int $FSNAME.$1 + destroy_pool_int $fsname.$poolname RC=$? [[ $RC -ne 0 ]] && return $RC - wait_update $HOSTNAME "lctl get_param -n lov.$FSNAME-*.pools.$1 \ - 2>/dev/null || echo foo" "foo" && return 0 + wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \ + 2>/dev/null || echo foo" "foo" || RC=1 + + if [[ $RC -eq 0 ]]; then + remove_pool_from_list $fsname.$poolname + else + error "destroy pool failed $1" + fi + return $RC +} + +destroy_pools () { + local fsname=${1:-$FSNAME} + local poolname + local listvar=${fsname}_CREATED_POOLS + + pool_list $fsname + + [ x${!listvar} = x ] && return 0 + + echo destroy the created pools: ${!listvar} + for poolname in ${!listvar//,/ }; do + destroy_pool $fsname.$poolname + done +} + +cleanup_pools () { + local fsname=${1:-$FSNAME} + trap 0 + destroy_pools $fsname } gather_logs () {