From d595ac0d3ba7f393a3d01672d8fe4b7cdd7700b0 Mon Sep 17 00:00:00 2001 From: Yu Jian Date: Sun, 9 Oct 2011 14:36:22 +0800 Subject: [PATCH] LU-690 test: wait_osc_import_state() fixes b=24498 wait_osc_import_state() fixes i=Vladimir.Saveliev -- increase maxtime to wait the timeout of 1st request; take into account at_min value; -- cleanup wait_osc_import_state() to use _wait_import_state(); -- ost-pools test_1 fix: use local var instead of global NAME Signed-off-by: Elena Gryaznova Signed-off-by: Yu Jian Change-Id: I36b163a291d783b1d772f05898ccee783d815298 Reviewed-on: http://review.whamcloud.com/1499 Tested-by: Hudson Tested-by: Maloo Reviewed-by: Minh Diep Reviewed-by: Oleg Drokin --- lustre/tests/ost-pools.sh | 6 +- lustre/tests/test-framework.sh | 123 +++++++++++++++++++++++++++-------------- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/lustre/tests/ost-pools.sh b/lustre/tests/ost-pools.sh index 4fbb75f..3e9afb9 100644 --- a/lustre/tests/ost-pools.sh +++ b/lustre/tests/ost-pools.sh @@ -255,9 +255,9 @@ test_1() { create_pool_fail p1234567891234567 echo "Creating a pool with a 1000 character pool name; should fail" - NAME="p" - for i in `seq 1 999`; do NAME=${NAME}"o"; done - create_pool_fail $NAME + local pool_name="p" + for i in $(seq 1 999); do pool_name=${pool_name}"o"; done + create_pool_fail $pool_name echo "pool_new should fail if fs-name or poolname are missing." do_facet $SINGLEMDS lctl pool_new .pool1 2>/dev/null diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 8ff0903..cba9fd0 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1604,6 +1604,7 @@ fail_abort() { local facet=$1 stop $facet change_active $facet + wait_for_facet $facet mount_facet $facet -o abort_recovery clients_up || echo "first df failed: $?" clients_up || error "post-failover df: $?" @@ -2739,15 +2740,18 @@ at_is_enabled() { fi } -at_max_get() { +at_get() { local facet=$1 + local at=$2 - # suppose that all ost-s has the same at_max set - if [ $facet == "ost" ]; then - do_facet ost1 "lctl get_param -n at_max" - else - do_facet $facet "lctl get_param -n at_max" - fi + # suppose that all ost-s have the same $at value set + [ $facet != "ost" ] || facet=ost1 + + do_facet $facet "lctl get_param -n $at" +} + +at_max_get() { + at_get $1 at_max } at_max_set() { @@ -2755,20 +2759,17 @@ at_max_set() { shift local facet + local hosts for facet in $@; do if [ $facet == "ost" ]; then - for i in `seq $OSTCOUNT`; do - do_facet ost$i "lctl set_param at_max=$at_max" - - done + facet=$(get_facets OST) elif [ $facet == "mds" ]; then - for i in `seq $MDSCOUNT`; do - do_facet mds$i "lctl set_param at_max=$at_max" - done - else - do_facet $facet "lctl set_param at_max=$at_max" + facet=$(get_facets MDS) fi + hosts=$(expand_list $hosts $(facets_hosts $facet)) done + + do_nodes $hosts lctl set_param at_max=$at_max } ################################## @@ -3852,12 +3853,19 @@ get_osc_import_name() { _wait_import_state () { local expected=$1 local CONN_PROC=$2 - local maxtime=${3:-max_recovery_time} + local maxtime=${3:-$(max_recovery_time)} local CONN_STATE local i=0 CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2) while [ "${CONN_STATE}" != "${expected}" ]; do + if [ "${expected}" == "DISCONN" ]; then + # for disconn we can check after proc entry is removed + [ "x${CONN_STATE}" == "x" ] && return 0 + # with AT enabled, we can have connect request timeout near of + # reconnect timeout and test can't see real disconnect + [ "${CONN_STATE}" == "CONNECTING" ] && return 0 + fi [ $i -ge $maxtime ] && \ error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}" && \ return 1 @@ -3873,43 +3881,78 @@ _wait_import_state () { wait_import_state() { local state=$1 local params=$2 - local maxtime=${3:-max_recovery_time} + local maxtime=${3:-$(max_recovery_time)} local param for param in ${params//,/ }; do _wait_import_state $state $param $maxtime || return done } + +# One client request could be timeouted because server was not ready +# when request was sent by client. +# The request timeout calculation details : +# ptl_send_rpc () +# /* We give the server rq_timeout secs to process the req, and +# add the network latency for our local timeout. */ +# request->rq_deadline = request->rq_sent + request->rq_timeout + +# ptlrpc_at_get_net_latency(request) ; +# +# ptlrpc_connect_import () +# request->rq_timeout = INITIAL_CONNECT_TIMEOUT +# +# init_imp_at () -> +# -> at_init(&at->iat_net_latency, 0, 0) -> iat_net_latency=0 +# ptlrpc_at_get_net_latency(request) -> +# at_get (max (iat_net_latency=0, at_min)) = at_min +# +# i.e.: +# request->rq_timeout + ptlrpc_at_get_net_latency(request) = +# INITIAL_CONNECT_TIMEOUT + at_min +# +# We will use obd_timeout instead of INITIAL_CONNECT_TIMEOUT +# because we can not get this value in runtime, +# the value depends on configure options, and it is not stored in /proc. +# obd_support.h: +# #define CONNECTION_SWITCH_MIN 5U +# #ifndef CRAY_XT3 +# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20) +# #else +# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/2) + +request_timeout () { + local facet=$1 + + # request->rq_timeout = INITIAL_CONNECT_TIMEOUT + local init_connect_timeout=$TIMEOUT + [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5 + + local at_min=$(at_get $facet at_min) + + echo $(( init_connect_timeout + at_min )) +} + wait_osc_import_state() { local facet=$1 local ost_facet=$2 local expected=$3 local ost=$(get_osc_import_name $facet $ost_facet) - local CONN_PROC - local CONN_STATE - local i=0 - CONN_PROC="osc.${ost}.ost_server_uuid" - CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2) - while [ "${CONN_STATE}" != "${expected}" ]; do - if [ "${expected}" == "DISCONN" ]; then - # for disconn we can check after proc entry is removed - [ "x${CONN_STATE}" == "x" ] && return 0 - # with AT we can have connect request timeout ~ reconnect timeout - # and test can't see real disconnect - [ "${CONN_STATE}" == "CONNECTING" ] && return 0 - fi - # disconnect rpc should be wait not more obd_timeout - [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \ - error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1 - sleep 1 - CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2) - i=$(($i + 1)) - done + local param="osc.${ost}.ost_server_uuid" + + # 1. wait the deadline of client 1st request (it could be skipped) + # 2. wait the deadline of client 2nd request + local maxtime=$(( 2 * $(request_timeout $facet))) + + if ! do_rpc_nodes $(facet_host $facet) \ +_wait_import_state $expected $param $maxtime; then + error "import is not in ${expected} state" + return 1 + fi - log "${ost_facet} now in ${CONN_STATE} state" return 0 } + get_clientmdc_proc_path() { echo "${1}-mdc-*" } @@ -4137,7 +4180,7 @@ do_ls () { max_recovery_time () { local init_connect_timeout=$(( TIMEOUT / 20 )) - [[ $init_connect_timeout > 5 ]] || init_connect_timeout=5 + [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5 local service_time=$(( $(at_max_get client) + $(( 2 * $(( 25 + 1 + init_connect_timeout)) )) )) -- 1.8.3.1