X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=95c89d8c99f714c88ea92f644f72576fc4a99ade;hp=8ff09036e4d1a20903fa6ef2156b6f5f6b5815df;hb=17a69cf25ed0991e04d85c259f4294dc59734e1e;hpb=f426fb920e3b97298a603a142d56be89c12f2006

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 8ff0903..95c89d8 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -139,7 +139,7 @@ init_test_env() {
         export LOGDIR=${LOGDIR:-${TMP}/test_logs/}/$(date +%s)
         export LOGDIRSET=true
     fi
-    export HOSTNAME=${HOSTNAME:-`hostname`}
+    export HOSTNAME=${HOSTNAME:-$(hostname -s)}
     if ! echo $PATH | grep -q $LUSTRE/utils; then
         export PATH=$LUSTRE/utils:$PATH
     fi
@@ -691,7 +691,7 @@ stop() {
     local mntpt=$(facet_mntpt $facet)
     running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true
     if [ ${running} -ne 0 ]; then
-        echo "Stopping $mntpt (opts:$@)"
+        echo "Stopping $mntpt (opts:$@) on $HOST"
         do_facet ${facet} umount -d $@ $mntpt
     fi
 
@@ -1517,6 +1517,14 @@ facet_failover() {
 
     echo "Failing $facet on node $host"
 
+    # Make sure the client data is synced to disk. LU-924
+    #
+    # We don't write client data synchrnously (to avoid flooding sync writes
+    # when there are many clients connecting), so if the server reboots before
+    # the client data reachs disk, the client data will be lost and the client
+    # will be evicted after recovery, which is not what we expected.
+    do_facet $facet "sync; sync; sync"
+
     local affected=$(affected_facets $facet)
 
     shutdown_facet $facet
@@ -1545,7 +1553,7 @@ obd_name() {
 
 replay_barrier() {
     local facet=$1
-    do_facet $facet sync
+    do_facet $facet "sync; sync; sync"
     df $MOUNT
 
     # make sure there will be no seq change
@@ -1561,7 +1569,7 @@ replay_barrier() {
 
 replay_barrier_nodf() {
     local facet=$1    echo running=${running}
-    do_facet $facet sync
+    do_facet $facet "sync; sync; sync"
     local svc=${facet}_svc
     echo Replay barrier on ${!svc}
     do_facet $facet $LCTL --device %${!svc} notransno
@@ -1604,6 +1612,7 @@ fail_abort() {
     local facet=$1
     stop $facet
     change_active $facet
+    wait_for_facet $facet
     mount_facet $facet -o abort_recovery
     clients_up || echo "first df failed: $?"
     clients_up || error "post-failover df: $?"
@@ -1614,6 +1623,18 @@ do_lmc() {
     exit 1
 }
 
+host_nids_address() {
+    local nodes=$1
+    local kind=$2
+
+    if [ -n "$kind" ]; then
+        nids=$(do_nodes $nodes "$LCTL list_nids | grep $kind | cut -f 1 -d '@'")
+    else
+        nids=$(do_nodes $nodes "$LCTL list_nids all | cut -f 1 -d '@'")
+    fi
+    echo $nids
+}
+
 h2name_or_ip() {
     if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else
         echo $1"@$2"
@@ -1654,6 +1675,95 @@ h2o2ib() {
 }
 declare -fx h2o2ib
 
+# This enables variables in cfg/"setup".sh files to support the pdsh HOSTLIST
+# expressions format. As a bonus we can then just pass in those variables
+# to pdsh. What this function does is take a HOSTLIST type string and
+# expand it into a space deliminated list for us.
+hostlist_expand() {
+    local hostlist=$1
+    local offset=$2
+    local myList
+    local item
+    local list
+
+    [ -z "$hostlist" ] && return
+
+    # Translate the case of [..],..,[..] to [..] .. [..]
+    list="${hostlist/],/] }"
+    front=${list%%[*}
+    [[ "$front" == *,* ]] && {
+        new="${list%,*} "
+        old="${list%,*},"
+        list=${list/${old}/${new}}
+    }
+
+    for item in $list; do
+        # Test if we have any []'s at all
+        if [ "$item" != "${item/\[/}" ]; then {
+            # Expand the [*] into list
+            name=${item%%[*}
+            back=${item#*]}
+
+            if [ "$name" != "$item" ]; then
+                group=${item#$name[*}
+                group=${group%%]*}
+
+                for range in ${group//,/ }; do
+                    begin=${range%-*}
+                    end=${range#*-}
+
+                    # Number of leading zeros
+                    padlen=${#begin}
+                    padlen2=${#end}
+                    end=$(echo $end | sed 's/0*//')
+                    [[ -z "$end" ]] && end=0
+                    [[ $padlen2 -gt $padlen ]] && {
+                        [[ $padlen2 -eq ${#end} ]] && padlen2=0
+                        padlen=$padlen2
+                    }
+                    begin=$(echo $begin | sed 's/0*//')
+                    [ -z $begin ] && begin=0
+
+                    for num in $(seq -f "%0${padlen}g" $begin $end); do
+                        value="${name#*,}${num}${back}"
+                        [ "$value" != "${value/\[/}" ] && {
+                            value=$(hostlist_expand "$value")
+                        }
+                        myList="$myList $value"
+                    done
+                done
+            fi
+        } else {
+            myList="$myList $item"
+        } fi
+    done
+    myList=${myList//,/ }
+    myList=${myList:1} # Remove first character which is a space
+
+    # Filter any duplicates without sorting
+    list="$myList "
+    myList="${list%% *}"
+
+    while [[ "$list" != ${myList##* } ]]; do
+        list=${list//${list%% *} /}
+        myList="$myList ${list%% *}"
+    done
+    myList="${myList%* }";
+
+    # We can select an object at a offset in the list
+    [ $# -eq 2 ] && {
+        cnt=0
+        for item in $myList; do
+            let cnt=cnt+1
+            [ $cnt -eq $offset ] && {
+                myList=$item
+            }
+        done
+        [ $(get_node_count $myList) -ne 1 ] && myList=""
+    }
+    echo $myList
+}
+
 facet_host() {
     local facet=$1
 
@@ -2739,15 +2849,18 @@ at_is_enabled() {
     fi
 }
 
-at_max_get() {
+at_get() {
     local facet=$1
+    local at=$2
 
-    # suppose that all ost-s has the same at_max set
-    if [ $facet == "ost" ]; then
-        do_facet ost1 "lctl get_param -n at_max"
-    else
-        do_facet $facet "lctl get_param -n at_max"
-    fi
+    # suppose that all ost-s have the same $at value set
+    [ $facet != "ost" ] || facet=ost1
+
+    do_facet $facet "lctl get_param -n $at"
+}
+
+at_max_get() {
+    at_get $1 at_max
 }
 
 at_max_set() {
@@ -2755,20 +2868,17 @@ at_max_set() {
     shift
 
     local facet
+    local hosts
     for facet in $@; do
         if [ $facet == "ost" ]; then
-            for i in `seq $OSTCOUNT`; do
-                do_facet ost$i "lctl set_param at_max=$at_max"
-
-            done
+            facet=$(get_facets OST)
         elif [ $facet == "mds" ]; then
-            for i in `seq $MDSCOUNT`; do
-                do_facet mds$i "lctl set_param at_max=$at_max"
-            done
-        else
-            do_facet $facet "lctl set_param at_max=$at_max"
+            facet=$(get_facets MDS)
         fi
+        hosts=$(expand_list $hosts $(facets_hosts $facet))
     done
+
+    do_nodes $hosts lctl set_param at_max=$at_max
 }
 
 ##################################
@@ -3307,7 +3417,17 @@ osc_to_ost()
 
 ostuuid_from_index()
 {
-    $LFS osts $2 | awk '/^'$1'/ { print $2 }'
+    $LFS osts $2 | sed -ne "/^$1: /s/.* \(.*\) .*$/\1/p"
+}
+
+ostname_from_index() {
+    local uuid=$(ostuuid_from_index $1)
+    echo ${uuid/_UUID/}
+}
+
+index_from_ostuuid()
+{
+    $LFS osts $2 | sed -ne "/${1}/s/\(.*\): .* .*$/\1/p"
 }
 
 remote_node () {
@@ -3436,19 +3556,20 @@ remote_nodes_list () {
 
 init_clients_lists () {
     # Sanity check: exclude the local client from RCLIENTS
-    local rclients=$(echo " $RCLIENTS " | sed -re "s/\s+$HOSTNAME\s+/ /g")
+    local clients=$(hostlist_expand "$RCLIENTS")
+    local rclients=$(exclude_items_from_list "$clients" $HOSTNAME)
 
     # Sanity check: exclude the dup entries
-    rclients=$(for i in $rclients; do echo $i; done | sort -u)
+    RCLIENTS=$(for i in ${rclients//,/ }; do echo $i; done | sort -u)
 
-    local clients="$SINGLECLIENT $HOSTNAME $rclients"
+    clients="$SINGLECLIENT $HOSTNAME $RCLIENTS"
 
     # Sanity check: exclude the dup entries from CLIENTS
     # for those configs which has SINGLCLIENT set to local client
     clients=$(for i in $clients; do echo $i; done | sort -u)
 
-    CLIENTS=`comma_list $clients`
-    local -a remoteclients=($rclients)
+    CLIENTS=$(comma_list $clients)
+    local -a remoteclients=($RCLIENTS)
     for ((i=0; $i<${#remoteclients[@]}; i++)); do
             varname=CLIENT$((i + 2))
             eval $varname=${remoteclients[i]}
@@ -3779,10 +3900,7 @@ convert_facet2label() {
 }
 
 get_clientosc_proc_path() {
-    local ost=$1
-
-    # exclude -osc-M*
-    echo "${1}-osc-[!M]*"
+    echo "${1}-osc-[^M]*"
 }
 
 get_lustre_version () {
@@ -3852,12 +3970,19 @@ get_osc_import_name() {
 _wait_import_state () {
     local expected=$1
     local CONN_PROC=$2
-    local maxtime=${3:-max_recovery_time}
+    local maxtime=${3:-$(max_recovery_time)}
     local CONN_STATE
     local i=0
 
     CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
     while [ "${CONN_STATE}" != "${expected}" ]; do
+        if [ "${expected}" == "DISCONN" ]; then
+            # for disconn we can check after proc entry is removed
+            [ "x${CONN_STATE}" == "x" ] && return 0
+            #  with AT enabled, we can have connect request timeout near of
+            # reconnect timeout and test can't see real disconnect
+            [ "${CONN_STATE}" == "CONNECTING" ] && return 0
+        fi
         [ $i -ge $maxtime ] && \
             error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}" && \
             return 1
@@ -3873,43 +3998,78 @@ _wait_import_state () {
 wait_import_state() {
     local state=$1
     local params=$2
-    local maxtime=${3:-max_recovery_time}
+    local maxtime=${3:-$(max_recovery_time)}
     local param
 
     for param in ${params//,/ }; do
         _wait_import_state $state $param $maxtime || return
     done
 }
+
+# One client request could be timed out because server was not ready
+# when request was sent by client.
+# The request timeout calculation details :
+# ptl_send_rpc ()
+#      /* We give the server rq_timeout secs to process the req, and
+#      add the network latency for our local timeout. */
+#      request->rq_deadline = request->rq_sent + request->rq_timeout +
+#           ptlrpc_at_get_net_latency(request) ;
+#
+# ptlrpc_connect_import ()
+#      request->rq_timeout = INITIAL_CONNECT_TIMEOUT
+#
+# init_imp_at () ->
+#   -> at_init(&at->iat_net_latency, 0, 0) -> iat_net_latency=0
+# ptlrpc_at_get_net_latency(request) ->
+#       at_get (max (iat_net_latency=0, at_min)) = at_min
+#
+# i.e.:
+# request->rq_timeout + ptlrpc_at_get_net_latency(request) =
+# INITIAL_CONNECT_TIMEOUT + at_min
+#
+# We will use obd_timeout instead of INITIAL_CONNECT_TIMEOUT
+# because we can not get this value in runtime,
+# the value depends on configure options, and it is not stored in /proc.
+# obd_support.h:
+# #define CONNECTION_SWITCH_MIN 5U
+# #ifndef CRAY_XT3
+# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/20)
+# #else
+# #define INITIAL_CONNECT_TIMEOUT max(CONNECTION_SWITCH_MIN,obd_timeout/2)
+
+request_timeout () {
+    local facet=$1
+
+    # request->rq_timeout = INITIAL_CONNECT_TIMEOUT
+    local init_connect_timeout=$TIMEOUT
+    [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5
+
+    local at_min=$(at_get $facet at_min)
+
+    echo $(( init_connect_timeout + at_min ))
+}
+
 wait_osc_import_state() {
     local facet=$1
     local ost_facet=$2
     local expected=$3
     local ost=$(get_osc_import_name $facet $ost_facet)
-    local CONN_PROC
-    local CONN_STATE
-    local i=0
 
-    CONN_PROC="osc.${ost}.ost_server_uuid"
-    CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
-    while [ "${CONN_STATE}" != "${expected}" ]; do
-        if [ "${expected}" == "DISCONN" ]; then 
-            # for disconn we can check after proc entry is removed
-            [ "x${CONN_STATE}" == "x" ] && return 0
-            #  with AT we can have connect request timeout ~ reconnect timeout
-            # and test can't see real disconnect
-            [ "${CONN_STATE}" == "CONNECTING" ] && return 0
-        fi
-        # disconnect rpc should be wait not more obd_timeout
-        [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
-            error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
-        sleep 1
-        CONN_STATE=$(do_facet $facet lctl get_param -n $CONN_PROC 2>/dev/null | cut -f2)
-        i=$(($i + 1))
-    done
+    local param="osc.${ost}.ost_server_uuid"
+
+    # 1. wait the deadline of client 1st request (it could be skipped)
+    # 2. wait the deadline of client 2nd request
+    local maxtime=$(( 2 * $(request_timeout $facet)))
+
+    if ! do_rpc_nodes $(facet_host $facet) \
+                _wait_import_state $expected $param $maxtime; then
+        error "import is not in ${expected} state"
+        return 1
+    fi
 
-    log "${ost_facet} now in ${CONN_STATE} state"
     return 0
 }
+
 get_clientmdc_proc_path() {
     echo "${1}-mdc-*"
 }
@@ -4137,7 +4297,7 @@ do_ls () {
 
 max_recovery_time () {
     local init_connect_timeout=$(( TIMEOUT / 20 ))
-    [[ $init_connect_timeout > 5 ]] || init_connect_timeout=5 
+    [[ $init_connect_timeout -ge 5 ]] || init_connect_timeout=5
 
     local service_time=$(( $(at_max_get client) + $(( 2 * $(( 25 + 1  + init_connect_timeout)) )) ))
 
@@ -4540,7 +4700,7 @@ remove_mdt_files() {
     local mdtdev=$2
     shift 2
     local files="$@"
-    local mntpt=${MOUNT%/*}/$facet
+    local mntpt=$(facet_mntpt $facet)
 
     echo "removing files from $mdtdev on $facet: $files"
     mount -t $FSTYPE $MDS_MOUNT_OPTS $mdtdev $mntpt || return $?
@@ -4557,7 +4717,7 @@ duplicate_mdt_files() {
     local mdtdev=$2
     shift 2
     local files="$@"
-    local mntpt=${MOUNT%/*}/$facet
+    local mntpt=$(facet_mntpt $facet)
 
     echo "duplicating files on $mdtdev on $facet: $files"
     mkdir -p $mntpt || return $?