LU-14271 tests: add new node crash method

Patch adds a new node crash method: node crash
is simulated by command executed on the victim nodes.
Example:
  POWER_DOWN=sysrqcrash
triggers a crash  on all victim nodes.

Test-Parameters: trivial
Signed-off-by: Elena Gryaznova <elena.gryaznova@hpe.com>
HPE-bug-id: LUS-8162
Reviewed-by: Vladimir Saveliev <vladimir.saveliev@hpe.com>
Reviewed-by: Sergey Cheremencev <sergey.cheremencev@hpe.com>
Tested-by: Vitaly Fertman <vitaly.fertman@hpe.com>
Change-Id: Id39b0f58fbb59236b3af51516fe968c888b04a17
Reviewed-on: https://review.whamcloud.com/41071
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh
index f4c4bb9..c26689d 100755 (executable)
--- a/lustre/tests/ha.sh
+++ b/lustre/tests/ha.sh
@@ -824,6 +824,20 @@ ha_powermanage()
        return 0
 }
 
        return 0
 }
 

+ha_power_down_cmd_fn()
+{
+       local nodes=$1
+       local cmd
+
+       case $ha_power_down_cmd in
+       # format is: POWER_DOWN=sysrqcrash
+       sysrqcrash) cmd="pdsh -S -w $nodes 'echo c > /proc/sysrq-trigger' &" ;;
+       *) cmd="$ha_power_down_cmd $nodes" ;;
+       esac
+
+       eval $cmd
+}
+

 ha_power_down()
 {
        local nodes=$1
 ha_power_down()
 {
        local nodes=$1


@@ -833,7 +847,7 @@ ha_power_down()
 
        case $ha_power_down_cmd in
                *pm*) state=off ;;
 
        case $ha_power_down_cmd in
                *pm*) state=off ;;

-               *sysrq* ) state=off ;;
+               sysrqcrash) state=off ;;

                *) state=on;;
        esac
 
                *) state=on;;
        esac
 


@@ -845,7 +859,7 @@ ha_power_down()
        ha_info "Powering down $nodes : cmd: $ha_power_down_cmd"
        for (( i=0; i<10; i++ )) {
                ha_info "attempt: $i"
        ha_info "Powering down $nodes : cmd: $ha_power_down_cmd"
        for (( i=0; i<10; i++ )) {
                ha_info "attempt: $i"

-               $ha_power_down_cmd $nodes &&
+               ha_power_down_cmd_fn $nodes &&

                        ha_powermanage $nodes $state && rc=0 && break
                sleep $ha_power_delay
        }
                        ha_powermanage $nodes $state && rc=0 && break
                sleep $ha_power_delay
        }