X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fha.sh;h=1360d0c36cfe8b34db4dd9952e921940b8ff089c;hp=bd47b7fedd0da9b80ea59c05ef7c13da39712a77;hb=c6928f0e479f1eed7b4c3f2373ac1e71bb812d59;hpb=47ca85aeab63c3d29aeef4931e605c177b475447

diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh
index bd47b7f..1360d0c 100755
--- a/lustre/tests/ha.sh
+++ b/lustre/tests/ha.sh
@@ -176,9 +176,13 @@ declare -a  ha_status_files
 declare     ha_machine_file=$ha_tmp_dir/machine_file
 declare     ha_power_down_cmd=${POWER_DOWN:-"pm -0"}
 declare     ha_power_up_cmd=${POWER_UP:-"pm -1"}
+declare     ha_power_delay=${POWER_DELAY:-60}
 declare     ha_failback_delay=${DELAY:-5}
 declare     ha_failback_cmd=${FAILBACK:-""}
 declare     ha_stripe_params=${STRIPEPARAMS:-"-c 0"}
+declare     ha_dir_stripe_count=${DSTRIPECOUNT:-"1"}
+declare     ha_mdt_index=${MDTINDEX:-"0"}
+declare     ha_mdt_index_random=${MDTINDEXRAND:-false}
 declare -a  ha_clients
 declare -a  ha_servers
 declare -a  ha_victims
@@ -355,18 +359,21 @@ ha_unlock()
 
 ha_dump_logs()
 {
-    local nodes=${1// /,}
-    local file=/tmp/$(basename $0)-$$-$(date +%s).dk
-    local lock=$ha_tmp_dir/lock-dump-logs
+	local nodes=${1// /,}
+	local file=/tmp/$(basename $0)-$$-$(date +%s).dk
+	local lock=$ha_tmp_dir/lock-dump-logs
+	local rc=0
 
-    ha_lock "$lock"
-    ha_info "Dumping lctl log to $file"
+	ha_lock "$lock"
+	ha_info "Dumping lctl log to $file"
 
 	#
 	# some nodes could crash, so
 	# do not exit with error if not all logs are dumped
 	#
-	ha_on $nodes "lctl dk >$file" ||
+	ha_on $nodes "lctl dk >>$file" || rc=$?
+
+	[ $rc -eq 0 ] ||
 		ha_error "not all logs are dumped! Some nodes are unreachable."
 	ha_unlock "$lock"
 }
@@ -383,6 +390,7 @@ ha_repeat_mpi_load()
 	local log=$ha_tmp_dir/$client-$tag
 	local rc=0
 	local nr_loops=0
+	local avg_loop_time=0
 	local start_time=$(date +%s)
 
 	cmd=${cmd//"{}"/$dir}
@@ -393,7 +401,14 @@ ha_repeat_mpi_load()
 	local machines="-machinefile $ha_machine_file"
 	while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do
 		{
-		ha_on $client mkdir -p "$dir" &&
+		local mdt_index
+		if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then
+			mdt_index=$(ha_rand $ha_mdt_index)
+		else
+			mdt_index=$ha_mdt_index
+		fi
+		ha_on $client $LFS mkdir -i$mdt_index -c$ha_dir_stripe_count "$dir" &&
+		ha_on $client $LFS getdirstripe "$dir" &&
 		ha_on $client chmod a+xwr $dir &&
 		ha_on $client "su mpiuser sh -c \" $mpirun $ha_mpirun_options \
 			-np $((${#ha_clients[@]} * mpi_threads_per_client )) \
@@ -413,7 +428,8 @@ ha_repeat_mpi_load()
 		nr_loops=$((nr_loops + 1))
 	done
 
-	avg_loop_time=$((($(date +%s) - start_time) / nr_loops))
+	[ $nr_loops -ne 0 ] &&
+		avg_loop_time=$((($(date +%s) - start_time) / nr_loops))
 
 	ha_info "$tag stopped: rc $rc avg loop time $avg_loop_time"
 }
@@ -461,16 +477,17 @@ ha_start_mpi_loads()
 
 ha_repeat_nonmpi_load()
 {
-    local client=$1
-    local load=$2
-    local status=$3
-    local tag=${ha_nonmpi_load_tags[$load]}
-    local cmd=${ha_nonmpi_load_cmds[$load]}
-    local dir=$ha_test_dir/$client-$tag
-    local log=$ha_tmp_dir/$client-$tag
-    local rc=0
-    local nr_loops=0
-    local start_time=$(date +%s)
+	local client=$1
+	local load=$2
+	local status=$3
+	local tag=${ha_nonmpi_load_tags[$load]}
+	local cmd=${ha_nonmpi_load_cmds[$load]}
+	local dir=$ha_test_dir/$client-$tag
+	local log=$ha_tmp_dir/$client-$tag
+	local rc=0
+	local nr_loops=0
+	local avg_loop_time=0
+	local start_time=$(date +%s)
 
     cmd=${cmd//"{}"/$dir}
 
@@ -491,9 +508,10 @@ ha_repeat_nonmpi_load()
 		nr_loops=$((nr_loops + 1))
 	done
 
-    avg_loop_time=$((($(date +%s) - start_time) / nr_loops))
+	[ $nr_loops -ne 0 ] &&
+		avg_loop_time=$((($(date +%s) - start_time) / nr_loops))
 
-    ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s"
+	ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s"
 }
 
 ha_start_nonmpi_loads()
@@ -570,17 +588,31 @@ ha_wait_loads()
 ha_power_down()
 {
 	local nodes=$1
+	local rc=1
+	local i
 
 	ha_info "Powering down $nodes"
-	$ha_power_down_cmd $nodes
+	for i in $(seq 1 5); do
+		$ha_power_down_cmd $nodes && rc=0 && break
+		sleep $ha_power_delay
+	done
+
+	[ $rc -eq 0 ] || ha_info "Failed Powering down in $i attempts"
 }
 
 ha_power_up()
 {
 	local nodes=$1
+	local rc=1
+	local i
 
 	ha_info "Powering up $nodes"
-	$ha_power_up_cmd $nodes
+	for i in $(seq 1 5); do
+		$ha_power_up_cmd $nodes && rc=0 && break
+		sleep $ha_power_delay
+	done
+
+	[ $rc -eq 0 ] || ha_info "Failed Powering up in $i attempts"
 }
 
 #