LU-11503 tests: fix "division by 0" defect

if stop file is created before ha_repeat_mpi_load() or
ha_repeat_nonmpi_load() are started nr_loops is equal to 0
and test fails with
   (1476438773 - start_time) / nr_loops: division by 0
Patch fixes this case.

Patch correctly processes ha_dump_logs()->ha_on() rc
to avoid the test failure when nodes are not accessible for
logs dumping.

Test-Parameters: trivial
Signed-off-by: Elena Gryaznova <c17455@cray.com>
Cray-bug-id: LUS-2205
Reviewed-by: Andrew Perepechko <c17827@cray.com>
Reviewed-by: Alexander Boyko <c17825@cray.com>
Change-Id: I97957875685492ddec66daa7c353ff6ef930933c
Reviewed-on: https://review.whamcloud.com/33350
Tested-by: Jenkins
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Alexandr Boyko <c17825@cray.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh
index 713811b..1360d0c 100755 (executable)
--- a/lustre/tests/ha.sh
+++ b/lustre/tests/ha.sh
@@ -359,18 +359,21 @@ ha_unlock()
 
 ha_dump_logs()
 {
 
 ha_dump_logs()
 {

-    local nodes=${1// /,}
-    local file=/tmp/$(basename $0)-$$-$(date +%s).dk
-    local lock=$ha_tmp_dir/lock-dump-logs
+       local nodes=${1// /,}
+       local file=/tmp/$(basename $0)-$$-$(date +%s).dk
+       local lock=$ha_tmp_dir/lock-dump-logs
+       local rc=0

 
 

-    ha_lock "$lock"
-    ha_info "Dumping lctl log to $file"
+       ha_lock "$lock"
+       ha_info "Dumping lctl log to $file"

 
        #
        # some nodes could crash, so
        # do not exit with error if not all logs are dumped
        #
 
        #
        # some nodes could crash, so
        # do not exit with error if not all logs are dumped
        #

-       ha_on $nodes "lctl dk >$file" ||
+       ha_on $nodes "lctl dk >>$file" || rc=$?
+
+       [ $rc -eq 0 ] ||

                ha_error "not all logs are dumped! Some nodes are unreachable."
        ha_unlock "$lock"
 }
                ha_error "not all logs are dumped! Some nodes are unreachable."
        ha_unlock "$lock"
 }


@@ -387,6 +390,7 @@ ha_repeat_mpi_load()
        local log=$ha_tmp_dir/$client-$tag
        local rc=0
        local nr_loops=0
        local log=$ha_tmp_dir/$client-$tag
        local rc=0
        local nr_loops=0

+       local avg_loop_time=0

        local start_time=$(date +%s)
 
        cmd=${cmd//"{}"/$dir}
        local start_time=$(date +%s)
 
        cmd=${cmd//"{}"/$dir}


@@ -424,7 +428,8 @@ ha_repeat_mpi_load()
                nr_loops=$((nr_loops + 1))
        done
 
                nr_loops=$((nr_loops + 1))
        done
 

-       avg_loop_time=$((($(date +%s) - start_time) / nr_loops))
+       [ $nr_loops -ne 0 ] &&
+               avg_loop_time=$((($(date +%s) - start_time) / nr_loops))

 
        ha_info "$tag stopped: rc $rc avg loop time $avg_loop_time"
 }
 
        ha_info "$tag stopped: rc $rc avg loop time $avg_loop_time"
 }


@@ -472,16 +477,17 @@ ha_start_mpi_loads()
 
 ha_repeat_nonmpi_load()
 {
 
 ha_repeat_nonmpi_load()
 {

-    local client=$1
-    local load=$2
-    local status=$3
-    local tag=${ha_nonmpi_load_tags[$load]}
-    local cmd=${ha_nonmpi_load_cmds[$load]}
-    local dir=$ha_test_dir/$client-$tag
-    local log=$ha_tmp_dir/$client-$tag
-    local rc=0
-    local nr_loops=0
-    local start_time=$(date +%s)
+       local client=$1
+       local load=$2
+       local status=$3
+       local tag=${ha_nonmpi_load_tags[$load]}
+       local cmd=${ha_nonmpi_load_cmds[$load]}
+       local dir=$ha_test_dir/$client-$tag
+       local log=$ha_tmp_dir/$client-$tag
+       local rc=0
+       local nr_loops=0
+       local avg_loop_time=0
+       local start_time=$(date +%s)

 
     cmd=${cmd//"{}"/$dir}
 
 
     cmd=${cmd//"{}"/$dir}
 


@@ -502,9 +508,10 @@ ha_repeat_nonmpi_load()
                nr_loops=$((nr_loops + 1))
        done
 
                nr_loops=$((nr_loops + 1))
        done
 

-    avg_loop_time=$((($(date +%s) - start_time) / nr_loops))
+       [ $nr_loops -ne 0 ] &&
+               avg_loop_time=$((($(date +%s) - start_time) / nr_loops))

 
 

-    ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s"
+       ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s"

 }
 
 ha_start_nonmpi_loads()
 }
 
 ha_start_nonmpi_loads()