From d292fb12a0bdcb3d8dae22767929b20b10d1197e Mon Sep 17 00:00:00 2001
From: Elena Gryaznova <elena.gryaznova@hpe.com>
Date: Thu, 20 Feb 2020 21:50:44 +0300
Subject: [PATCH] LU-13275 tests: enhance racer with failover

Patch adds the possibilty to run facet_failovier <random_facet>
if RACER_FAILOVER is set to true.
The type of facets are set over FAIL_TARGETS variable,
default value is "MDS OST".

Test-Parameters: trivial \
envdefinitions=RACER_FAILOVER=true testlist=racer
Cray-bug-id: LUS-8280
Signed-off-by: Elena Gryaznova <c17455@cray.com>
Reviewed-by: Vladimir Saveliev <vladimir.saveliev@hpe.com>
Reviewed-by: Andriy Skulysh <andriy.skulysh@hpe.com>
Change-Id: Iacb6ac855bf77d7c6930fdbb7ec192ecc46264b3
Reviewed-on: https://review.whamcloud.com/37652
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Reviewed-by: Vladimir Saveliev <c17830@cray.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
---
 lustre/tests/racer.sh | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/lustre/tests/racer.sh b/lustre/tests/racer.sh
index f3e855f..892e9ea 100644
--- a/lustre/tests/racer.sh
+++ b/lustre/tests/racer.sh
@@ -26,6 +26,18 @@ CLIENTS=${CLIENTS:-$HOSTNAME}
 RACERDIRS=${RACERDIRS:-"$DIR $DIR2"}
 echo RACERDIRS=$RACERDIRS
 
+RACER_FAILOVER=${RACER_FAILOVER:-false}
+FAIL_TARGETS=${FAIL_TARGETS:-"MDS OST"}
+RACER_FAILOVER_PERIOD=${RACER_FAILOVER_PERIOD:-60}
+
+if $RACER_FAILOVER; then
+	declare -a  victims
+	for target in $FAIL_TARGETS; do
+		victims=(${victims[@]} $(get_facets $target))
+	done
+	echo Victim facets ${victims[@]}
+fi
+
 #LU-4684
 RACER_ENABLE_MIGRATION=false
 
@@ -57,6 +69,18 @@ RACER_ENABLE_PFL=${RACER_ENABLE_PFL:-true}
 RACER_ENABLE_DOM=${RACER_ENABLE_DOM:-true}
 RACER_ENABLE_FLR=${RACER_ENABLE_FLR:-true}
 
+fail_random_facet () {
+	local facets=${victims[@]}
+	facets=${facets// /,}
+
+	sleep $RACER_FAILOVER_PERIOD
+	while [ ! -f $racer_done ]; do
+		local facet=$(get_random_entry $facets)
+		facet_failover $facet
+		sleep $RACER_FAILOVER_PERIOD
+	done
+}
+
 # run racer
 test_1() {
 	local rrc=0
@@ -64,6 +88,9 @@ test_1() {
 	local clients=$CLIENTS
 	local RDIRS
 	local i
+	local racer_done=$TMP/racer_done
+
+	rm -f $racer_done
 
 	for d in ${RACERDIRS}; do
 		is_mounted $d || continue
@@ -100,6 +127,13 @@ test_1() {
 		rpids="$rpids $pid"
 	done
 
+	local failpid=""
+	if $RACER_FAILOVER; then
+		fail_random_facet &
+		failpid=$!
+		echo racers failpid: $failpid
+	fi
+
 	local lss_pids=""
 	if $RACER_ENABLE_SNAPSHOT; then
 		lss_gen_conf
@@ -123,6 +157,12 @@ test_1() {
 		fi
 	done
 
+	if $RACER_FAILOVER; then
+		touch $racer_done
+		wait $failpid
+		rrc=$((rrc + $?))
+	fi
+
 	if $RACER_ENABLE_SNAPSHOT; then
 		killall -q lss_create.sh
 		killall -q lss_destroy.sh
-- 
1.8.3.1