local facet=$1
# Use default policy if $2 is not passed by caller.
- #define OBD_RECOVERY_TIMEOUT (obd_timeout * 5 / 2)
- # as we are in process of changing obd_timeout in different ways
- # let's set MAX longer than that
- local MAX=${2:-$(( TIMEOUT * 4 ))}
+ local MAX=${2:-$(max_recovery_time)}
local var_svc=${facet}_svc
local procfile="*.${!var_svc}.recovery_status"
rm -rf $LOGDIR/err
echo
+ log_sub_test_begin test_${1}
(run_one $1 "$2") 2>&1 | tee $test_log
local RC=${PIPESTATUS[0]}
duration=$((`date +%s` - $BEFORE))
pass "(${duration}s)"
[ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err)
- log_sub_test test_${1} $TEST_STATUS $duration "$RC" "$TEST_ERROR"
+ log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR"
if [ -f $LOGDIR/err ]; then
$FAIL_ON_ERROR && exit $RC
}
wait_import_state () {
- local facet=$1
- local expected=$2
- local CONN_PROC=$3
+ local expected=$1
+ local CONN_PROC=$2
local CONN_STATE
local i=0
- CONN_STATE=$(do_facet $facet $LCTL get_param -n $CONN_PROC | awk '/state/ {print $2}')
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
while [ "${CONN_STATE}" != "${expected}" ]; do
if [ "${expected}" == "DISCONN" ]; then
# for disconn we can check after proc entry is removed
[ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
error "can't put import for $CONN_PROC into ${expected} state" && return 1
sleep 1
- CONN_STATE=$(do_facet $facet $LCTL get_param -n $CONN_PROC | awk '/state/ {print $2}')
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
i=$(($i + 1))
done
local ost_facet=$2
local expected=$3
local ost=$(get_osc_import_name $facet $ost_facet)
- local CONN_PROC="osc.${ost}.import"
+ local CONN_PROC
+ local CONN_STATE
+ local i=0
- wait_import_state $facet $expected $CONN_PROC || return 1
+ CONN_PROC="osc.${ost}.ost_server_uuid"
+ CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ while [ "${CONN_STATE}" != "${expected}" ]; do
+ if [ "${expected}" == "DISCONN" ]; then
+ # for disconn we can check after proc entry is removed
+ [ "x${CONN_STATE}" == "x" ] && return 0
+ # with AT we can have connect request timeout ~ reconnect timeout
+ # and test can't see real disconnect
+ [ "${CONN_STATE}" == "CONNECTING" ] && return 0
+ fi
+ # disconnect rpc should be wait not more obd_timeout
+ [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
+ error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
+ sleep 1
+ CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ i=$(($i + 1))
+ done
+
+ log "${ost_facet} now in ${CONN_STATE} state"
return 0
}
-
get_clientmdc_proc_path() {
- local mdc=$(convert_facet2label $1)
-
- echo "${mdc}-mdc-*"
-}
-
-wait_mdc_import_state() {
- local facet=$1
- local expected=$2
- local mdc=$(get_clientmdc_proc_path $facet)
- local CONN_PROC="mdc.${mdc}.import"
-
- wait_import_state client $expected $CONN_PROC || return 1
- return 0
+ echo "${1}-mdc-*"
}
do_rpc_nodes () {
do_nodesv $list "PATH=$RPATH sh rpc.sh $@ "
}
-wait_client_import_state () {
- local facet=$1
- local expected=$2
+wait_clients_import_state () {
+ local list=$1
+ local facet=$2
+ local expected=$3
shift
+ local label=$(convert_facet2label $facet)
+ local proc_path
case $facet in
- ost* ) wait_osc_import_state client $facet $expected || return 1;;
- mds* ) wait_mdc_import_state $facet $expected || return 1 ;;
- * ) error "unknown facet!"
- return 1 ;;
+ ost* ) proc_path="osc.$(get_clientosc_proc_path $label).ost_server_uuid" ;;
+ mds* ) proc_path="mdc.$(get_clientmdc_proc_path $label).mds_server_uuid" ;;
+ *) error "unknown facet!" ;;
esac
- return 0
-}
-wait_clients_import_state () {
- local list=$1
- shift
-
- if ! do_rpc_nodes $list wait_client_import_state "$@"; then
- error "import is not in expected state"
+ if ! do_rpc_nodes $list wait_import_state $expected $proc_path; then
+ error "import is not in ${expected} state"
return 1
fi
- return 0
}
oos_full() {
return $rc
}
+# target_start_and_reset_recovery_timer()
+# service_time = at_est2timeout(service_time);
+# service_time += 2 * (CONNECTION_SWITCH_MAX + CONNECTION_SWITCH_INC +
+# INITIAL_CONNECT_TIMEOUT);
+# CONNECTION_SWITCH_MAX : min(25U, max(CONNECTION_SWITCH_MIN,obd_timeout))
+#define CONNECTION_SWITCH_INC 1
+#define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20)
+#define CONNECTION_SWITCH_MIN 5U
+
+max_recovery_time () {
+ local init_connect_timeout=$(( TIMEOUT / 20 ))
+ [[ $init_connect_timeout > 5 ]] || init_connect_timeout=5
+
+ local service_time=$(( $(at_max_get client) + $(( 2 * $(( 25 + 1 + init_connect_timeout)) )) ))
+
+ echo $service_time
+}
+
get_clients_mount_count () {
local clients=${CLIENTS:-`hostname`}
R=0
else
N=$((N + 1))
- while [ $N -le $M ]; do
+ while [ $N -lt $M ]; do
R=$((R * N))
N=$((N + 1))
done
return 0
else
echo "found $res $flavor connections of $dir, not ready ($expect)"
+ return 0
sleep 4
fi
done
yml_log_test $1 >> $YAML_LOG
}
-log_sub_test() {
- yml_log_sub_test $@ >> $YAML_LOG
+log_sub_test_begin() {
+ yml_log_sub_test_begin $@ >> $YAML_LOG
+}
+
+log_sub_test_end() {
+ yml_log_sub_test_end $@ >> $YAML_LOG
}
run_llverdev()