Fix for conf-sanity test_50e, implement correct waiting.

Original patch by Alexey Lyashkov.

b=19280
i=robert.read

diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index db65e25..789ee7e 100644 (file)
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -66,7 +66,7 @@ gen_config() {
        # and stop to generate the startup logs. 
        start_mds
        start_ost
-       sleep 5
+        wait_osc_import_state mds ost FULL
        stop_ost
        stop_mds
 }
@@ -557,6 +557,7 @@ run_test 20 "remount ro,rw mounts work and doesn't break /etc/mtab"
 test_21a() {
         start_mds
        start_ost
+        wait_osc_import_state mds ost FULL
        stop_ost
        stop_mds
 }
@@ -565,6 +566,7 @@ run_test 21a "start mds before ost, stop ost first"
 test_21b() {
         start_ost
        start_mds
+        wait_osc_import_state mds ost FULL
        stop_mds
        stop_ost
 }
@@ -574,6 +576,7 @@ test_21c() {
         start_ost
        start_mds
        start_ost2
+        wait_osc_import_state mds ost2 FULL
        stop_ost
        stop_ost2
        stop_mds
@@ -588,8 +591,7 @@ test_22() {
        echo Client mount with ost in logs, but none running
        start_ost
        # wait until mds connected to ost and open client connection
-       # ping_interval + 1
-       sleep $((TIMEOUT / 4 + 1))
+        wait_osc_import_state mds ost FULL
        stop_ost
        mount_client $MOUNT
        # check_mount will block trying to contact ost
@@ -1585,6 +1587,7 @@ test_46a() {
        start_mds || return 1
        #first client should see only one ost
        start_ost || return 2
+        wait_osc_import_state mds ost FULL
        #start_client
        mount_client $MOUNT || return 3
        
@@ -1593,9 +1596,12 @@ test_46a() {
        start ost4 `ostdevname 4` $OST_MOUNT_OPTS || return 6
        start ost5 `ostdevname 5` $OST_MOUNT_OPTS || return 7
        # wait until ost2-5 is sync
-       # ping_interval + 1
-       sleep $((TIMEOUT / 4 + 1))
-       #second client see both ost's
+        # ping_interval + 1
+        wait_osc_import_state mds ost2 FULL
+        wait_osc_import_state mds ost3 FULL
+        wait_osc_import_state mds ost4 FULL
+        wait_osc_import_state mds ost5 FULL
+       #second client see all ost's
 
        mount_client $MOUNT2 || return 8
        $LFS setstripe $MOUNT2 -c -1 || return 9
@@ -1795,12 +1801,8 @@ test_50b() {
 
        # Wait for client to detect down OST
        stop_ost || error "Unable to stop OST1"
-       CONN_PROC="osc.$FSNAME-OST0000-osc-[M]*.ost_server_uuid"
-       CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       while [ "${CONN_STATE}" = "FULL" ]; do
-               sleep 1
-               CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       done
+        wait_osc_import_state mds ost DISCONN
+
        lazystatfs $MOUNT || error "lazystatfs should don't have returned EIO"
 
        umount_client $MOUNT || error "Unable to unmount client"
@@ -1818,12 +1820,7 @@ test_50c() {
 
        # Wait for client to detect down OST
        stop_ost || error "Unable to stop OST1"
-       CONN_PROC="osc.$FSNAME-OST0000-osc-[M]*.ost_server_uuid"
-       CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       while [ ${CONN_STATE} = "FULL" ]; do
-               sleep 1
-               CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       done
+        wait_osc_import_state mds ost DISCONN
        lazystatfs $MOUNT || error "lazystatfs failed with one down server"
 
        umount_client $MOUNT || error "Unable to unmount client"
@@ -1855,30 +1852,19 @@ run_test 50d "lazystatfs client/server conn race =========================="
 test_50e() {
        local RC1
        local pid
-       CONN_PROC="osc.$FSNAME-OST0000-osc-[M]*.ost_server_uuid"
 
        reformat_and_config
        start_mds || return 1
        #first client should see only one ost
        start_ost || return 2
-       CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       while [ "${CONN_STATE}" != "FULL" ]; do
-               sleep 1
-               CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       done
-
-       lctl set_param llite.$FSNAME-*.lazystatfs=0
+        wait_osc_import_state mds ost FULL
 
        # Wait for client to detect down OST
        stop_ost || error "Unable to stop OST1"
+        wait_osc_import_state mds ost DISCONN
 
-       CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       while [ "${CONN_STATE}" = "FULL" ]; do
-               sleep 1
-               CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       done
-       
        mount_client $MOUNT || error "Unable to mount client"
+        lctl set_param llite.$FSNAME-*.lazystatfs=0
        
        multiop_bg_pause $MOUNT _f
        RC1=$?
@@ -1909,25 +1895,17 @@ test_50f() {
        start_mds || error "Unable to start mds"
        #first client should see only one ost
        start_ost || error "Unable to start OST1"
-       start_ost2 || error "Unable to start OST2"
-       CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       while [ "${CONN_STATE}" != "FULL" ]; do
-               sleep 1
-               CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       done
+        wait_osc_import_state mds ost FULL
 
-       lctl set_param llite.$FSNAME-*.lazystatfs=0
+        start_ost2 || error "Unable to start OST2"
+        wait_osc_import_state mds ost2 FULL
 
        # Wait for client to detect down OST
        stop_ost2 || error "Unable to stop OST2"
 
-       CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       while [ "${CONN_STATE}" = "FULL" ]; do
-               sleep 1
-               CONN_STATE=`lctl get_param -n $CONN_PROC | cut -f2`
-       done
-       
+       wait_osc_import_state mds ost2 DISCONN
        mount_client $MOUNT || error "Unable to mount client"
+        lctl set_param llite.$FSNAME-*.lazystatfs=0
        
        multiop_bg_pause $MOUNT _f
        RC1=$?

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index b8c16a5..eda0aec 100644 (file)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -2675,3 +2675,87 @@ delayed_recovery_enabled () {
     do_facet $SINGLEMDS lctl get_param -n mdd.${!var}.stale_export_age > /dev/null 2>&1
 }
 
+########################
+convert_facet2name() {
+    case "$1" in
+        "ost" ) echo "OST0000" ;;
+        "ost1") echo "OST0000" ;;
+        "ost2") echo "OST0001" ;;
+        "ost3") echo "OST0002" ;;
+        "ost4") echo "OST0003" ;;
+        "ost5") echo "OST0004" ;;
+        *) error "unknown facet!" ;;
+    esac
+}
+
+get_clientosc_proc_path() {
+    local ost=$1
+
+    echo "{$1}-osc-*"
+}
+
+get_lustre_version () {
+    local node=${1:-"mds"}    
+    do_facet $node $LCTL get_param -n version |  awk '/^lustre:/ {print $2}'
+}
+
+get_mds_version_major () {
+    local version=$(get_lustre_version mds)
+    echo $version | awk -F. '{print $1}'
+}
+
+get_mds_version_minor () {
+    local version=$(get_lustre_version mds)
+    echo $version | awk -F. '{print $2}'
+}
+
+get_mdtosc_proc_path() {
+    local ost=$1
+    local major=$(get_mds_version_major)
+    local minor=$(get_mds_version_minor)
+    if [ $major -le 1 -a $minor -le 8 ] ; then
+        echo "${ost}-osc"
+    else
+        echo "${ost}-osc-MDT0000"
+    fi
+}
+
+get_osc_import_name() {
+    local node=$1
+    local ost=$2
+    local name=$(convert_facet2name $ost)
+
+    if [ "$node" == "mds" ]; then
+        get_mdtosc_proc_path $name
+        return 0
+    fi
+
+    get_clientosc_proc_path $name
+    return 0
+}
+
+wait_osc_import_state() {
+    local node=$1
+    local ost_facet=$2
+    local expected=$3
+    local ost=$(get_osc_import_name $node $ost_facet)
+    local CONN_PROC
+    local CONN_STATE
+    local i=0
+
+    CONN_PROC="osc.${FSNAME}-${ost}.ost_server_uuid"
+    CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC | cut -f2)
+    while [ "${CONN_STATE}" != "${expected}" ]; do
+        # for disconn we can check after proc entry is removed
+        [ "x${CONN_STATE}" == "x" -a "${expected}" == "DISCONN" ] && return 0
+        # disconnect rpc should be wait not more obd_timeout
+        [ $i -ge $(($TIMEOUT * 3 / 2)) ] && \
+            error "can't put import for ${ost}(${ost_facet}) into ${expected} state" && return 1
+        sleep 1
+        CONN_STATE=$(do_facet $node lctl get_param -n $CONN_PROC | cut -f2)
+        i=$(($i + 1))
+    done
+
+    log "${ost_facet} now in ${CONN_STATE} state"
+    return 0
+}