X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=cba9fd0e168bb4fc15e5a98bb545414e0375cd30;hp=a28f1f6293b5e8b1118b43b3147af71c0dfccc93;hb=d595ac0d3ba7f393a3d01672d8fe4b7cdd7700b0;hpb=c2751b31e55518d1791cd5b87adc842f4fbbee83

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index a28f1f6..cba9fd0 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -330,6 +330,20 @@ load_module() {
     fi
 }
 
+llite_lloop_enabled() {
+    local n1=$(uname -r | cut -d. -f1)
+    local n2=$(uname -r | cut -d. -f2)
+    local n3=$(uname -r | cut -d- -f1 | cut -d. -f3)
+
+    # load the llite_lloop module for < 2.6.32 kernels
+    if [[ $n1 -lt 2 ]] || [[ $n1 -eq 2 && $n2 -lt 6 ]] || \
+       [[ $n1 -eq 2 && $n2 -eq 6 && $n3 -lt 32 ]] || \
+        $LOAD_LLOOP; then
+        return 0
+    fi
+    return 1
+}
+
 load_modules_local() {
     if [ -n "$MODPROBE" ]; then
         # use modprobe
@@ -372,21 +386,9 @@ load_modules_local() {
         load_module obdfilter/obdfilter
     fi
 
-    load_module_llite_lloop() {
-        local n1=$(uname -r | cut -d. -f1)
-        local n2=$(uname -r | cut -d. -f2)
-        local n3=$(uname -r | cut -d- -f1 | cut -d. -f3)
-
-    # load the llite_lloop module for < 2.6.32 kernels
-        if [[ $n1 -lt 2 ]] || [[ $n1 -eq 2 && $n2 -lt 6 ]] || \
-            [[ $n1 -eq 2 && $n2 -eq 6 && $n3 -lt 32 ]] || \
-            $LOAD_LLOOP; then
-                load_module llite/llite_lloop
-        fi
-    }
 
     load_module llite/lustre
-    load_module_llite_lloop
+    llite_lloop_enabled && load_module llite/llite_lloop
     [ -d /r ] && OGDB=${OGDB:-"/r/tmp"}
     OGDB=${OGDB:-$TMP}
     rm -f $OGDB/ogdb-$HOSTNAME
@@ -1602,6 +1604,7 @@ fail_abort() {
     local facet=$1
     stop $facet
     change_active $facet
+    wait_for_facet $facet
     mount_facet $facet -o abort_recovery
     clients_up || echo "first df failed: $?"
     clients_up || error "post-failover df: $?"
@@ -1711,6 +1714,8 @@ change_active() {
     # save the active host for this facet
     local activevar=${facet}active
     echo "$activevar=${!activevar}" > $TMP/$activevar
+    [[ $facet = mds1 ]] && combined_mgs_mds && \
+        echo "mgsactive=${!activevar}" > $TMP/mgsactive
     local TO=`facet_active_host $facet`
     echo "Failover $facet to $TO"
     done
@@ -1840,6 +1845,7 @@ add() {
     # make sure its not already running
     stop ${facet} -f
     rm -f $TMP/${facet}active
+    [[ $facet = mds1 ]] && combined_mgs_mds && rm -f $TMP/mgsactive
     do_facet ${facet} $MKFS $*
 }
 
@@ -1861,6 +1867,8 @@ mdsdevname() {
 
 facet_mntpt () {
     local facet=$1
+    [[ $facet = mgs ]] && combined_mgs_mds && facet="mds1"
+
     local var=${facet}_MOUNT
     eval mntpt=${!var:-${MOUNT%/*}/$facet}
 
@@ -1892,6 +1900,7 @@ stopall() {
         stop mds$num -f
         rm -f ${TMP}/mds${num}active
     done
+    combined_mgs_mds && rm -f $TMP/mgsactive
 
     for num in `seq $OSTCOUNT`; do
         stop ost$num -f
@@ -2731,15 +2740,18 @@ at_is_enabled() {
     fi
 }
 
-at_max_get() {
+at_get() {
     local facet=$1
+    local at=$2
 
-    # suppose that all ost-s has the same at_max set
-    if [ $facet == "ost" ]; then
-        do_facet ost1 "lctl get_param -n at_max"
-    else
-        do_facet $facet "lctl get_param -n at_max"
-    fi
+    # suppose that all ost-s have the same $at value set
+    [ $facet != "ost" ] || facet=ost1
+
+    do_facet $facet "lctl get_param -n $at"
+}
+
+at_max_get() {
+    at_get $1 at_max
 }
 
 at_max_set() {
@@ -2747,20 +2759,17 @@ at_max_set() {
     shift
 
     local facet
+    local hosts
     for facet in $@; do
         if [ $facet == "ost" ]; then
-            for i in `seq $OSTCOUNT`; do
-                do_facet ost$i "lctl set_param at_max=$at_max"
-
-            done
+            facet=$(get_facets OST)
         elif [ $facet == "mds" ]; then
-            for i in `seq $MDSCOUNT`; do
-                do_facet mds$i "lctl set_param at_max=$at_max"
-            done
-        else
-            do_facet $facet "lctl set_param at_max=$at_max"
+            facet=$(get_facets MDS)
         fi
+        hosts=$(expand_list $hosts $(facets_hosts $facet))
     done
+
+    do_nodes $hosts lctl set_param at_max=$at_max
 }
 
 ##################################
@@ -3844,12 +3853,19 @@ get_osc_import_name() {
 _wait_import_state () {
     local expected=$1
     local CONN_PROC=$2
-    local maxtime=${3:-max_recovery_time}
+    local maxtime=${3:-$(max_recovery_time)}
     local CONN_STATE
     local i=0
 
     CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
     while [ "${CONN_STATE}" != "${expected}" ]; do
+        if [ "${expected}" == "DISCONN" ]; then
+            # for disconn we can check after proc entry is removed
+            [ "x${CONN_STATE}" == "x" ] && return 0
+            #  with AT enabled, we can have connect request timeout near of
+            # reconnect timeout and test can't see real disconnect
+            [ "${CONN_STATE}" == "CONNECTING" ] && return 0
+        fi
         [ $i -ge $maxtime ] && \
             error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}" && \
             return 1
@@ -3865,43 +3881,78 @@ _wait_import_state () {
 wait_import_state() {
     local state=$1
     local params=$2
-    local maxtime=${3:-max_recovery_time}
+    local maxtime=${3:-$(max_recovery_time)}
     local param
 
     for param in ${params//,/ }; do
         _wait_import_state $state $param $maxtime || return
     done
 }
+
+# One client request could be timeouted because server was not ready
+# when request was sent by client.
+# The request timeout calculation details :
+# ptl_send_rpc ()
+#      /* We give the server rq_timeout secs to process the req, and
+#      add the network latency for our local timeout. */
+#      request->rq_deadline = request->rq_sent + request->rq_timeout +
+#           ptlrpc_at_get_net_latency(request) ;
+#
+# ptlrpc_connect_import ()
+#      request->rq_timeout = INITIAL_CONNECT_TIMEOUT
+#
+# init_imp_at () ->
+#   -> at_init(&at->iat_net_latency, 0, 0) -> iat_net_latency=0
+# ptlrpc_at_get_net_latency(request) ->
+#       at_get (max (iat_net_latency=0, at_min)) = at_min
+#
+# i.e.:
+# request->rq_timeout + ptlrpc_at_get_net_latency(request) =
+# INITIAL_CONNECT_TIMEOUT + at_min
+#
+# We will use obd_timeout instead of INITIAL_CONNECT_TIMEOUT
+# because we can not get this value in runtime,
+# the value depends on configure options, and it is not stored in /proc.
+# obd_support.h:
+# #define CONNECTION_SWITCH_MIN 5U
+# #ifndef CRAY_XT3
+# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20)
+# #else
+# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/2)
+
+request_timeout () {
+    local facet=$1
+
+    # request->rq_timeout = INITIAL_CONNECT_TIMEOUT
+    local init_connect_timeout=$TIMEOUT
+    [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5
+
+    local at_min=$(at_get $facet at_min)
+
+    echo $(( init_connect_timeout + at_min ))
+}
+
 wait_osc_import_state() {
     local facet=$1
     local ost_facet=$2
     local expected=$3
     local ost=$(get_osc_import_name $facet $ost_facet)
-    local CONN_PROC
-    local CONN_STATE
-    local i=0
 
-    CONN_PROC="osc.${ost}.ost_server_uuid"
-    CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
-    while [ "${CONN_STATE}" != "${expected}" ]; do
-        if [ "${expected}" == "DISCONN" ]; then 
-            # for disconn we can check after proc entry is removed
-            [ "x${CONN_STATE}" == "x" ] && return 0
-            #  with AT we can have connect request timeout ~ reconnect timeout
-            # and test can't see real disconnect
-            [ "${CONN_STATE}" == "CONNECTING" ] && return 0
-        fi
-        # disconnect rpc should be wait not more obd_timeout
-        [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
-            error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
-        sleep 1
-        CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
-        i=$(($i + 1))
-    done
+    local param="osc.${ost}.ost_server_uuid"
+
+    # 1. wait the deadline of client 1st request (it could be skipped)
+    # 2. wait the deadline of client 2nd request
+    local maxtime=$(( 2 * $(request_timeout $facet)))
+
+    if ! do_rpc_nodes $(facet_host $facet) \
+_wait_import_state $expected $param $maxtime; then
+        error "import is not in ${expected} state"
+        return 1
+    fi
 
-    log "${ost_facet} now in ${CONN_STATE} state"
     return 0
 }
+
 get_clientmdc_proc_path() {
     echo "${1}-mdc-*"
 }
@@ -4129,7 +4180,7 @@ do_ls () {
 
 max_recovery_time () {
     local init_connect_timeout=$(( TIMEOUT / 20 ))
-    [[ $init_connect_timeout > 5 ]] || init_connect_timeout=5 
+    [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5
 
     local service_time=$(( $(at_max_get client) + $(( 2 * $(( 25 + 1  + init_connect_timeout)) )) ))
 
@@ -4497,6 +4548,7 @@ log_sub_test_end() {
 run_llverdev()
 {
         local dev=$1
+        local llverdev_opts=$2
         local devname=$(basename $1)
         local size=$(grep "$devname"$ /proc/partitions | awk '{print $3}')
         # loop devices aren't in /proc/partitions
@@ -4506,10 +4558,24 @@ run_llverdev()
 
         local partial_arg=""
         # Run in partial (fast) mode if the size
-        # of a partition > 10 GB
-        [ $size -gt 10 ] && partial_arg="-p"
+        # of a partition > 1 GB
+        [ $size -gt 1 ] && partial_arg="-p"
+
+        llverdev --force $partial_arg $llverdev_opts $dev
+}
+
+run_llverfs()
+{
+        local dir=$1
+        local llverfs_opts=$2
+        local partial_arg=""
+        local size=$(df -B G $dir |tail -n 1 |awk '{print $2}' |sed 's/G//') #GB
+
+        # Run in partial (fast) mode if the size
+        # of a partition > 1 GB
+        [ $size -gt 1 ] && partial_arg="-p"
 
-        llverdev --force $partial_arg $dev
+        llverfs $partial_arg $llverfs_opts $dir
 }
 
 remove_mdt_files() {