From c03ef784aa9b899da4461c6dd30223121a248cf4 Mon Sep 17 00:00:00 2001
From: Andriy Skulysh <c17819@cray.com>
Date: Fri, 24 Jul 2020 12:07:27 +0300
Subject: [PATCH] LU-15474 test: facet_failover() should mount all facets
 concurrently

A single host can hold multiple facets.
They all are needed to be mounted after reboot.
Mounting all facets for in one by one mode increases
total recovery time and leads to evictions
while using striped files.

Change-Id: Ifa76dfd080cd89f1316ccb1d5552bb12d070168a
HPE-bug-id: LUS-9141
Reviewed-by: Vladimir Saveliev <c17830@cray.com>
Reviewed-by: Alexander Zarochentsev <c17826@cray.com>
Signed-off-by: Andriy Skulysh <andriy.skulysh@hpe.com>
Reviewed-on: https://review.whamcloud.com/46266
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Elena Gryaznova <elena.gryaznova@hpe.com>
Reviewed-by: Vladimir Saveliev <vladimir.saveliev@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---
 lustre/tests/test-framework.sh | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 1f7c6a3..04f9599 100755
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -1702,9 +1702,17 @@ set_hostid () {
 mount_facets () {
 	local facets=${1:-$(get_facets)}
 	local facet
+	local -a mountpids
+	local total=0
+	local ret=0
 
 	for facet in ${facets//,/ }; do
-		mount_facet $facet
+		mount_facet $facet &
+		mountpids[total]=$!
+		total=$((total+1))
+	done
+	for ((index=0; index<$total; index++)); do
+		wait ${mountpids[index]}
 		local RC=$?
 		[ $RC -eq 0 ] && continue
 
@@ -1714,8 +1722,9 @@ mount_facets () {
 		else
 			error "Restart of $facet failed!"
 		fi
-		return $RC
+		ret=$RC
 	done
+	return $ret
 }
 
 #
@@ -3704,6 +3713,8 @@ facet_failover() {
 	$E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
 		$(mdsdevname 1) "-n" || error "Running e2fsck")
 
+	local -a mountpids
+
 	for ((index=0; index<$total; index++)); do
 		facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
 		echo reboot facets: ${affecteds[index]}
@@ -3724,11 +3735,17 @@ facet_failover() {
 			affecteds[index]=$(exclude_items_from_list \
 				${affecteds[index]} mgs)
 		fi
-		# FIXME; has to be changed to mount all facets concurrently
 		if [ -n "${affecteds[index]}" ]; then
 			echo mount facets: ${affecteds[index]}
-			mount_facets ${affecteds[index]}
+			mount_facets ${affecteds[index]} &
+			mountpids[index]=$!
 		fi
+	done
+	for ((index=0; index<$total; index++)); do
+		if [ -n "${affecteds[index]}" ]; then
+			wait ${mountpids[index]}
+		fi
+
 		if $GSS_SK; then
 			do_nodes $(comma_list $(all_nodes)) \
 				"keyctl show | grep lustre | cut -c1-11 |
-- 
1.8.3.1