From: Mikhail Pershin Date: Mon, 5 Apr 2010 18:49:50 +0000 (-0700) Subject: b=20647 Ensure all needed pids exits before failover in test set 53 X-Git-Tag: 1.10.0.40~15 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=8f6d85eebc312b64d8e8a35b0be3ae137a50a45c;hp=1c68bf5b2df41f51ee5c69c7bd8cdf23234351f5 b=20647 Ensure all needed pids exits before failover in test set 53 Cleanup wait_..._import_state function. i=rread i=grev --- diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index 4b0f5c0..1e4e91f 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -1170,6 +1170,9 @@ test_53c() { kill -USR1 $close_pid cancel_lru_locks mdc # force the close + #bz20647: make sure all pids are exists before failover + [ -d /proc/$close_pid ] || error "close_pid doesn't exist" + [ -d /proc/$open_pid ] || error "open_pid doesn't exists" replay_barrier_nodf $SINGLEMDS fail_nodf $SINGLEMDS wait $open_pid || return 1 @@ -1263,6 +1266,9 @@ test_53f() { kill -USR1 $close_pid cancel_lru_locks mdc # force the close + #bz20647: make sure all pids are exists before failover + [ -d /proc/$close_pid ] || error "close_pid doesn't exist" + [ -d /proc/$open_pid ] || error "open_pid doesn't exists" replay_barrier_nodf $SINGLEMDS fail_nodf $SINGLEMDS wait $open_pid || return 1 @@ -1295,8 +1301,11 @@ test_53g() { do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000115" kill -USR1 $close_pid cancel_lru_locks mdc # force the close - do_facet $SINGLEMDS "lctl set_param fail_loc=0" + + #bz20647: make sure all pids are exists before failover + [ -d /proc/$close_pid ] || error "close_pid doesn't exist" + [ -d /proc/$open_pid ] || error "open_pid doesn't exists" replay_barrier_nodf $SINGLEMDS fail_nodf $SINGLEMDS wait $open_pid || return 1 @@ -1330,6 +1339,9 @@ test_53h() { cancel_lru_locks mdc # force the close sleep 1 + #bz20647: make sure all pids are exists before failover + [ -d /proc/$close_pid ] || error "close_pid doesn't exist" + [ -d /proc/$open_pid ] || error "open_pid doesn't exists" replay_barrier_nodf $SINGLEMDS fail_nodf $SINGLEMDS wait $open_pid || return 1 diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 772bcbf..0165fee 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -3312,12 +3312,13 @@ get_osc_import_name() { } wait_import_state () { - local expected=$1 - local CONN_PROC=$2 + local facet=$1 + local expected=$2 + local CONN_PROC=$3 local CONN_STATE local i=0 - CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2) + CONN_STATE=$(do_facet $facet $LCTL get_param -n $CONN_PROC | awk '/state/ {print $2}') while [ "${CONN_STATE}" != "${expected}" ]; do if [ "${expected}" == "DISCONN" ]; then # for disconn we can check after proc entry is removed @@ -3330,7 +3331,7 @@ wait_import_state () { [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \ error "can't put import for $CONN_PROC into ${expected} state" && return 1 sleep 1 - CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2) + CONN_STATE=$(do_facet $facet $LCTL get_param -n $CONN_PROC | awk '/state/ {print $2}') i=$(($i + 1)) done @@ -3343,33 +3344,26 @@ wait_osc_import_state() { local ost_facet=$2 local expected=$3 local ost=$(get_osc_import_name $facet $ost_facet) - local CONN_PROC - local CONN_STATE - local i=0 + local CONN_PROC="osc.${ost}.import" - CONN_PROC="osc.${ost}.ost_server_uuid" - CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2) - while [ "${CONN_STATE}" != "${expected}" ]; do - if [ "${expected}" == "DISCONN" ]; then - # for disconn we can check after proc entry is removed - [ "x${CONN_STATE}" == "x" ] && return 0 - # with AT we can have connect request timeout ~ reconnect timeout - # and test can't see real disconnect - [ "${CONN_STATE}" == "CONNECTING" ] && return 0 - fi - # disconnect rpc should be wait not more obd_timeout - [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \ - error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1 - sleep 1 - CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2) - i=$(($i + 1)) - done - - log "${ost_facet} now in ${CONN_STATE} state" + wait_import_state $facet $expected $CONN_PROC || return 1 return 0 } + get_clientmdc_proc_path() { - echo "${1}-mdc-*" + local mdc=$(convert_facet2label $1) + + echo "${mdc}-mdc-*" +} + +wait_mdc_import_state() { + local facet=$1 + local expected=$2 + local mdc=$(get_clientmdc_proc_path $facet) + local CONN_PROC="mdc.${mdc}.import" + + wait_import_state client $expected $CONN_PROC || return 1 + return 0 } do_rpc_nodes () { @@ -3381,24 +3375,29 @@ do_rpc_nodes () { do_nodesv $list "PATH=$RPATH sh rpc.sh $@ " } -wait_clients_import_state () { - local list=$1 - local facet=$2 - local expected=$3 +wait_client_import_state () { + local facet=$1 + local expected=$2 shift - local label=$(convert_facet2label $facet) - local proc_path case $facet in - ost* ) proc_path="osc.$(get_clientosc_proc_path $label).ost_server_uuid" ;; - mds* ) proc_path="mdc.$(get_clientmdc_proc_path $label).mds_server_uuid" ;; - *) error "unknown facet!" ;; + ost* ) wait_osc_import_state client $facet $expected || return 1;; + mds* ) wait_mdc_import_state $facet $expected || return 1 ;; + * ) error "unknown facet!" + return 1 ;; esac + return 0 +} + +wait_clients_import_state () { + local list=$1 + shift - if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then - error "import is not in ${expected} state" + if ! do_rpc_nodes $list wait_client_import_state "$@"; then + error "import is not in expected state" return 1 fi + return 0 } oos_full() {