LU-15474 test: facet_failover() should mount all facets concurrently

A single host can hold multiple facets.
They all are needed to be mounted after reboot.
Mounting all facets for in one by one mode increases
total recovery time and leads to evictions
while using striped files.

Change-Id: Ifa76dfd080cd89f1316ccb1d5552bb12d070168a
HPE-bug-id: LUS-9141
Reviewed-by: Vladimir Saveliev <c17830@cray.com>
Reviewed-by: Alexander Zarochentsev <c17826@cray.com>
Signed-off-by: Andriy Skulysh <andriy.skulysh@hpe.com>
Reviewed-on: https://review.whamcloud.com/46266
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Elena Gryaznova <elena.gryaznova@hpe.com>
Reviewed-by: Vladimir Saveliev <vladimir.saveliev@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 1f7c6a3..04f9599 100755 (executable)
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1702,9 +1702,17 @@ set_hostid () {
 mount_facets () {
        local facets=${1:-$(get_facets)}
        local facet
+       local -a mountpids
+       local total=0
+       local ret=0
 
        for facet in ${facets//,/ }; do
-               mount_facet $facet
+               mount_facet $facet &
+               mountpids[total]=$!
+               total=$((total+1))
+       done
+       for ((index=0; index<$total; index++)); do
+               wait ${mountpids[index]}
                local RC=$?
                [ $RC -eq 0 ] && continue
 
@@ -1714,8 +1722,9 @@ mount_facets () {
                else
                        error "Restart of $facet failed!"
                fi
-               return $RC
+               ret=$RC
        done
+       return $ret
 }
 
 #
@@ -3704,6 +3713,8 @@ facet_failover() {
        $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
                $(mdsdevname 1) "-n" || error "Running e2fsck")
 
+       local -a mountpids
+
        for ((index=0; index<$total; index++)); do
                facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
                echo reboot facets: ${affecteds[index]}
@@ -3724,11 +3735,17 @@ facet_failover() {
                        affecteds[index]=$(exclude_items_from_list \
                                ${affecteds[index]} mgs)
                fi
-               # FIXME; has to be changed to mount all facets concurrently
                if [ -n "${affecteds[index]}" ]; then
                        echo mount facets: ${affecteds[index]}
-                       mount_facets ${affecteds[index]}
+                       mount_facets ${affecteds[index]} &
+                       mountpids[index]=$!
                fi
+       done
+       for ((index=0; index<$total; index++)); do
+               if [ -n "${affecteds[index]}" ]; then
+                       wait ${mountpids[index]}
+               fi
+
                if $GSS_SK; then
                        do_nodes $(comma_list $(all_nodes)) \
                                "keyctl show | grep lustre | cut -c1-11 |