X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Fha.sh;h=352bdaafcea2606157f2b4d638fe934494a61468;hp=8f5551f7938c80b608f28f41f4a7ad23664976c1;hb=56f69854dae234fa974ab4f1dff909aea601a592;hpb=d3a044086f5790fec2747c653dca26b8ec529e2d diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh index 8f5551f..352bdaa 100755 --- a/lustre/tests/ha.sh +++ b/lustre/tests/ha.sh @@ -44,6 +44,8 @@ # Workloads dry run for several seconds; no failures will be introduced. # This option is useful to verify the loads. # -u is ignored in this case +# -m +# Reboot victim nodes simultaneously. # # # ASSUMPTIONS @@ -72,14 +74,17 @@ # applications are run in short loops so that their exit status can be waited # for and checked within reasonable time by ha_wait_loads. # The set of MPI and non-MPI workloads are configurable by parameters: -# ha_mpi_loads -# default set: dd, tar, iozone # ha_nonmpi_loads -# default set: ior, simul. +# default set: dd, tar, iozone +# ha_mpi_loads +# default set: ior, simul, mdtest # # The number of clients run MPI loads is configured by parameter # ha_mpi_instances. Only one client runs MPI workloads by default. # +# MPI workloads can be run from several users. The list of users to use is +# configured by parameter ha_mpi_users, default is "mpiuser". +# # PROCESS STRUCTURE AND IPC # # On the node where this script is run, the processes look like this: @@ -90,6 +95,8 @@ # ~ mpirun IOR # ~ ha.sh (ha_repeat_mpi_load simul) # ~ mpirun simul +# ~ ha.sh (ha_repeat_mpi_load mdtest) +# ~ mpirun mdtest # ~ ... (one for each MPI load) # # ~ ha.sh (ha_repeat_nonmpi_load client2 dbench) @@ -110,6 +117,7 @@ SIMUL=${SIMUL:-$(which simul 2> /dev/null || true)} IOR=${IOR:-$(which IOR 2> /dev/null || true)} +MDTEST=${MDTEST:-$(which mdtest 2> /dev/null || true)} ior_blockSize=${ior_blockSize:-6g} mpi_threads_per_client=${mpi_threads_per_client:-2} @@ -136,6 +144,21 @@ ha_info() echo "$0: $(date +%H:%M:%S' '%s):" "$@" } +ha_touch() +{ + local date=$(date +%H:%M:%S' '%s) + + [[ $1 =~ stop ]] && + echo $date ${FUNCNAME[1]} $2 >> $ha_stop_file || + true + [[ $1 =~ fail ]] && + echo $date ${FUNCNAME[1]} $2 >> $ha_fail_file || + true + [[ $1 =~ lfsck ]] && + echo $date ${FUNCNAME[1]} $2 >> $ha_lfsck_stop || + true +} + ha_log() { local nodes=${1// /,} @@ -163,69 +186,132 @@ ha_trap_err() trap ha_trap_err ERR set -eE +declare ha_power_down_pids declare ha_tmp_dir=/tmp/$(basename $0)-$$ declare ha_stop_file=$ha_tmp_dir/stop declare ha_fail_file=$ha_tmp_dir/fail +declare ha_pm_states=$ha_tmp_dir/ha_pm_states declare ha_status_file_prefix=$ha_tmp_dir/status declare -a ha_status_files declare ha_machine_file=$ha_tmp_dir/machine_file +declare ha_lfsck_log=$ha_tmp_dir/lfsck.log +declare ha_lfsck_lock=$ha_tmp_dir/lfsck.lock +declare ha_lfsck_stop=$ha_tmp_dir/lfsck.stop +declare ha_lfsck_bg=${LFSCK_BG:-false} +declare ha_lfsck_after=${LFSCK_AFTER:-false} +declare ha_lfsck_node=${LFSCK_NODE:-""} +declare ha_lfsck_device=${LFSCK_DEV:-""} +declare ha_lfsck_types=${LFSCK_TYPES:-"namespace layout"} +declare ha_lfsck_custom_params=${LFSCK_CUSTOM_PARAMS:-""} +declare ha_lfsck_wait=${LFSCK_WAIT:-1200} +declare ha_lfsck_fail_on_repaired=${LFSCK_FAIL_ON_REPAIRED:-false} declare ha_power_down_cmd=${POWER_DOWN:-"pm -0"} declare ha_power_up_cmd=${POWER_UP:-"pm -1"} +declare ha_power_delay=${POWER_DELAY:-60} +declare ha_node_up_delay=${NODE_UP_DELAY:-10} +declare ha_wait_nodes_up=${WAIT_NODES_UP:-600} +declare ha_pm_host=${PM_HOST:-$(hostname)} declare ha_failback_delay=${DELAY:-5} declare ha_failback_cmd=${FAILBACK:-""} declare ha_stripe_params=${STRIPEPARAMS:-"-c 0"} +declare ha_test_dir_stripe_count=${TDSTRIPECOUNT:-"1"} +declare ha_test_dir_mdt_index=${TDMDTINDEX:-"0"} +declare ha_test_dir_mdt_index_random=${TDMDTINDEXRAND:-false} +declare ha_dir_stripe_count=${DSTRIPECOUNT:-"1"} +declare ha_dir_stripe_count_random=${DSTRIPECOUNTRAND:-false} +declare ha_mdt_index=${MDTINDEX:-"0"} +declare ha_mdt_index_random=${MDTINDEXRAND:-false} declare -a ha_clients declare -a ha_servers declare -a ha_victims +declare -a ha_victims_pair declare ha_test_dir=/mnt/lustre/$(basename $0)-$$ +declare -a ha_testdirs=(${ha_test_dirs="$ha_test_dir"}) + +for ((i=0; i<${#ha_testdirs[@]}; i++)); do + echo I=$i ${ha_testdirs[i]} + ha_testdirs[i]="${ha_testdirs[i]}/$(basename $0)-$$" + echo i=$i ${ha_testdirs[i]} +done + +declare ha_cleanup=${CLEANUP:-true} declare ha_start_time=$(date +%s) declare ha_expected_duration=$((60 * 60 * 24)) declare ha_max_failover_period=10 declare ha_nr_loops=0 declare ha_stop_signals="SIGINT SIGTERM SIGHUP" -declare ha_load_timeout=$((60 * 10)) +declare ha_load_timeout=${LOAD_TIMEOUT:-$((60 * 10))} declare ha_workloads_only=false declare ha_workloads_dry_run=false +declare ha_simultaneous=false declare ha_mpi_instances=${ha_mpi_instances:-1} -declare ha_mpi_loads=${ha_mpi_loads="ior simul"} +declare ha_mpi_loads=${ha_mpi_loads="ior simul mdtest"} declare -a ha_mpi_load_tags=($ha_mpi_loads) +declare -a ha_mpiusers=(${ha_mpi_users="mpiuser"}) +declare -a ha_users +declare -A ha_mpiopts + +for ((i=0; i<${#ha_mpiusers[@]}; i++)); do + u=${ha_mpiusers[i]%%:*} + o="" + # user gets empty option if ha_mpi_users does not specify it explicitly + [[ ${ha_mpiusers[i]} =~ : ]] && o=${ha_mpiusers[i]##*:} + ha_users[i]=$u + ha_mpiopts[$u]+=" $o" +done +ha_users=(${!ha_mpiopts[@]}) declare ha_ior_params=${IORP:-'" -b $ior_blockSize -t 2m -w -W -T 1"'} declare ha_simul_params=${SIMULP:-'" -n 10"'} +declare ha_mdtest_params=${MDTESTP:-'" -i 1 -n 1000"'} declare ha_mpirun_options=${MPIRUN_OPTIONS:-""} +declare ha_clients_stripe=${CLIENTSSTRIPE:-'"$STRIPEPARAMS"'} +declare ha_nclientsset=${NCLIENTSSET:-1} +declare ha_ninstmustfail=${NINSTMUSTFAIL:-0} + +declare ha_racer_params=${RACERP:-"MDSCOUNT=1"} eval ha_params_ior=($ha_ior_params) eval ha_params_simul=($ha_simul_params) +eval ha_params_mdtest=($ha_mdtest_params) +eval ha_stripe_clients=($ha_clients_stripe) declare ha_nparams_ior=${#ha_params_ior[@]} declare ha_nparams_simul=${#ha_params_simul[@]} +declare ha_nparams_mdtest=${#ha_params_mdtest[@]} +declare ha_nstripe_clients=${#ha_stripe_clients[@]} declare -A ha_mpi_load_cmds=( - [ior]="$IOR -o {}/f.ior {params}" - [simul]="$SIMUL {params} -d {}" + [ior]="$IOR -o {}/f.ior {params}" + [simul]="$SIMUL {params} -d {}" + [mdtest]="$MDTEST {params} -d {}" ) +declare racer=${RACER:-"$(dirname $0)/racer/racer.sh"} + declare ha_nonmpi_loads=${ha_nonmpi_loads="dd tar iozone"} declare -a ha_nonmpi_load_tags=($ha_nonmpi_loads) -declare -a ha_nonmpi_load_cmds=( - "dd if=/dev/zero of={}/f.dd bs=1M count=256" - "tar cf - /etc | tar xf - -C {}" - "iozone -a -e -+d -s $iozone_SIZE {}/f.iozone" +declare -A ha_nonmpi_load_cmds=( + [dd]="dd if=/dev/zero of={}/f.dd bs=1M count=256" + [tar]="tar cf - /etc | tar xf - -C {}" + [iozone]="iozone -a -e -+d -s $iozone_SIZE {}/f.iozone" + [racer]="$ha_racer_params $racer {}" ) +declare ha_check_attrs="find {} -type f -ls 2>&1 | grep -e '?'" ha_usage() { - ha_info "Usage: $0 -c HOST[,...] -s HOST[,...]" \ - "-v HOST[,...] [-d DIRECTORY] [-u SECONDS]" + ha_info "Usage: $0 -c HOST[,...] -s HOST[,...]" \ + "-v HOST[,...] -f HOST[,...] [-d DIRECTORY] [-u SECONDS]" } ha_process_arguments() { local opt - while getopts hc:s:v:d:p:u:wr opt; do + while getopts hc:s:v:d:p:u:wrmf: opt; do case $opt in h) ha_usage @@ -255,6 +341,12 @@ ha_process_arguments() r) ha_workloads_dry_run=true ;; + m) + ha_simultaneous=true + ;; + f) + ha_victims_pair=(${OPTARG//,/ }) + ;; \?) ha_usage exit 1 @@ -288,29 +380,31 @@ ha_on() # -S is to be used here to track the # remote command return values # - pdsh -S -w $nodes PATH=/usr/local/sbin:/usr/local/bin:/sbin:\ -/bin:/usr/sbin:/usr/bin "$@" || + pdsh -S -w $nodes "PATH=/usr/local/sbin:/usr/local/bin:/sbin:\ +/bin:/usr/sbin:/usr/bin; $@" || rc=$? return $rc } ha_trap_exit() { - touch "$ha_stop_file" + ha_touch stop trap 0 if [ -e "$ha_fail_file" ]; then - ha_info "Test directory $ha_test_dir not removed" + ha_info "Test directories ${ha_testdirs[@]} not removed" ha_info "Temporary directory $ha_tmp_dir not removed" else - ha_on ${ha_clients[0]} rm -rf "$ha_test_dir" + $ha_cleanup && + ha_on ${ha_clients[0]} rm -rf ${ha_testdirs[@]} || + ha_info "Test directories ${ha_testdirs[@]} not removed" ha_info "Please find the results in the directory $ha_tmp_dir" fi } ha_trap_stop_signals() { - ha_info "${ha_stop_signals// /,} received" - touch "$ha_stop_file" + ha_info "${ha_stop_signals// /,} received" + ha_touch stop "${ha_stop_signals// /,} received" } ha_sleep() @@ -324,6 +418,15 @@ ha_sleep() sleep $n || true } +ha_wait_unlock() +{ + local lock=$1 + + while [ -e $lock ]; do + sleep 1 + done +} + ha_lock() { local lock=$1 @@ -342,18 +445,21 @@ ha_unlock() ha_dump_logs() { - local nodes=${1// /,} - local file=/tmp/$(basename $0)-$$-$(date +%s).dk - local lock=$ha_tmp_dir/lock-dump-logs + local nodes=${1// /,} + local file=/tmp/$(basename $0)-$$-$(date +%s).dk + local lock=$ha_tmp_dir/lock-dump-logs + local rc=0 - ha_lock "$lock" - ha_info "Dumping lctl log to $file" + ha_lock "$lock" + ha_info "Dumping lctl log to $file" # # some nodes could crash, so # do not exit with error if not all logs are dumped # - ha_on $nodes "lctl dk >$file" || + ha_on $nodes "lctl dk >>$file" || rc=$? + + [ $rc -eq 0 ] || ha_error "not all logs are dumped! Some nodes are unreachable." ha_unlock "$lock" } @@ -364,45 +470,96 @@ ha_repeat_mpi_load() local load=$2 local status=$3 local parameter=$4 + local machines=$5 + local stripeparams=$6 + local mpiuser=$7 + local mustpass=$8 + local mpirunoptions=$9 + local test_dir=${10} local tag=${ha_mpi_load_tags[$load]} local cmd=${ha_mpi_load_cmds[$tag]} - local dir=$ha_test_dir/$client-$tag + local dir=$test_dir/$client-$tag local log=$ha_tmp_dir/$client-$tag local rc=0 + local rccheck=0 local nr_loops=0 + local avg_loop_time=0 local start_time=$(date +%s) + local check_attrs=${ha_check_attrs//"{}"/$dir} cmd=${cmd//"{}"/$dir} cmd=${cmd//"{params}"/$parameter} + [[ -n "$ha_postcmd" ]] && ha_postcmd=${ha_postcmd//"{}"/$dir} + [[ -n "$ha_precmd" ]] && ha_precmd=${ha_precmd//"{}"/$dir} ha_info "Starting $tag" - local machines="-machinefile $ha_machine_file" - while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do + machines="-machinefile $machines" + while [ ! -e "$ha_stop_file" ] && ((rc == 0)) && ((rccheck == 0)); do + ha_info "$client Starts: $mpiuser: $cmd" 2>&1 | tee -a $log { - ha_on $client mkdir -p "$dir" && + local mdt_index + if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then + mdt_index=$(ha_rand $((ha_mdt_index + 1)) ) + else + mdt_index=$ha_mdt_index + fi + local dir_stripe_count + if $ha_dir_stripe_count_random && + [ $ha_dir_stripe_count -ne 1 ]; then + dir_stripe_count=$(($(ha_rand $ha_dir_stripe_count) + 1)) + else + dir_stripe_count=$ha_dir_stripe_count + fi + [[ -n "$ha_precmd" ]] && ha_info "$ha_precmd" && + ha_on $client "$ha_precmd" >>"$log" 2>&1 + ha_info "$client Creates $dir with -i$mdt_index -c$dir_stripe_count " + ha_on $client $LFS mkdir -i$mdt_index -c$dir_stripe_count "$dir" && + ha_on $client $LFS getdirstripe "$dir" && + ha_on $client $LFS setstripe $stripeparams $dir && + ha_on $client $LFS getstripe $dir && ha_on $client chmod a+xwr $dir && - ha_on $client "su mpiuser sh -c \" $mpirun $ha_mpirun_options \ - -np $((${#ha_clients[@]} * mpi_threads_per_client )) \ - $machines $cmd \" " && - ha_on $client rm -rf "$dir"; - } >>"$log" 2>&1 || rc=$? - - ha_info rc=$rc - - if ((rc != 0)); then - touch "$ha_fail_file" - touch "$ha_stop_file" + ha_on $client "su $mpiuser bash -c \" $mpirun $mpirunoptions \ + -np $((${#ha_clients[@]} * mpi_threads_per_client / ha_nclientsset)) \ + $machines $cmd \" " || rc=$? + ha_on ${ha_clients[0]} "$check_attrs && \ + $LFS df $dir && \ + $check_attrs " && rccheck=1 + [[ -n "$ha_postcmd" ]] && ha_info "$ha_postcmd" && + ha_on $client "$ha_postcmd" >>"$log" 2>&1 + (( ((rc == 0)) && ((rccheck == 0)) && (( mustpass != 0 )) )) || + (( ((rc != 0)) && ((rccheck == 0)) && (( mustpass == 0 )) )) && + ha_on $client rm -rf "$dir"; + } >>"$log" 2>&1 + + ha_info $client: rc=$rc rccheck=$rccheck mustpass=$mustpass + + # mustpass=0 means that failure is expected + if (( rccheck != 0 )); then + ha_touch stop,fail $client,$tag + ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" + elif (( rc !=0 )); then + if (( mustpass != 0 )); then + ha_touch stop,fail $client,$tag + ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" + else + # Ok to fail + rc=0 + fi + elif (( mustpass == 0 )); then + ha_touch stop,fail $client,$tag ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" fi - echo $rc >"$status" + echo rc=$rc rccheck=$rccheck mustpass=$mustpass >"$status" nr_loops=$((nr_loops + 1)) done - avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) + [ $nr_loops -ne 0 ] && + avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) - ha_info "$tag stopped: rc $rc avg loop time $avg_loop_time" + ha_info "$tag stopped: rc=$rc mustpass=$mustpass \ + avg loop time $avg_loop_time" } ha_start_mpi_loads() @@ -413,24 +570,49 @@ ha_start_mpi_loads() local status local n local nparam + local machines + local m + local -a mach + local mpiuser + local nmpi - for client in ${ha_clients[@]}; do - ha_info ha_machine_file=$ha_machine_file - echo $client >> $ha_machine_file + # ha_mpi_instances defines the number of + # clients start mpi loads; should be <= ${#ha_clients[@]} + # do nothing if + # ha_mpi_instances = 0 + # or + # ${#ha_mpi_load_tags[@]} =0 + local inst=$ha_mpi_instances + (( inst == 0 )) || (( ${#ha_mpi_load_tags[@]} == 0 )) && + ha_info "no mpi load to start" && + return 0 + + (( inst <= ${#ha_clients[@]} )) || inst=${#ha_clients[@]} + + # Define names for machinefiles for each client set + for (( n=0; n < $ha_nclientsset; n++ )); do + mach[$n]=$ha_machine_file$n + done + + for ((n = 0; n < ${#ha_clients[@]}; n++)); do + m=$(( n % ha_nclientsset)) + machines=${mach[m]} + ha_info machine_file=$machines + echo ${ha_clients[n]} >> $machines done local dirname=$(dirname $ha_machine_file) for client in ${ha_clients[@]}; do ha_on $client mkdir -p $dirname - scp $ha_machine_file $client:$ha_machine_file + scp $ha_machine_file* $client:$dirname done - # ha_mpi_instances defines the number of - # clients start mpi loads; should be <= ${#ha_clients[@]} - local inst=$ha_mpi_instances - (( inst <= ${#ha_clients[@]} )) || inst=${#ha_clients[@]} - + local ndir for ((n = 0; n < $inst; n++)); do client=${ha_clients[n]} + nmpi=$((n % ${#ha_users[@]})) + mpiuser=${ha_users[nmpi]} + ndir=$((n % ${#ha_testdirs[@]})) + test_dir=${ha_testdirs[ndir]} for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do tag=${ha_mpi_load_tags[$load]} status=$ha_status_file_prefix-$tag-$client @@ -440,7 +622,17 @@ ha_start_mpi_loads() nparam=$((n % num)) local aref=ha_params_$tag[nparam] local parameter=${!aref} - ha_repeat_mpi_load $client $load $status "$parameter" & + local nstripe=$((n % ha_nstripe_clients)) + aref=ha_stripe_clients[nstripe] + local stripe=${!aref} + local m=$(( n % ha_nclientsset)) + machines=${mach[m]} + local mustpass=1 + [[ $ha_ninstmustfail == 0 ]] || + mustpass=$(( n % ha_ninstmustfail )) + ha_repeat_mpi_load $client $load $status "$parameter" \ + $machines "$stripe" "$mpiuser" "$mustpass" \ + "${ha_mpiopts[$mpiuser]} $ha_mpirun_options" "$test_dir" & ha_status_files+=("$status") done done @@ -448,71 +640,225 @@ ha_start_mpi_loads() ha_repeat_nonmpi_load() { - local client=$1 - local load=$2 - local status=$3 - local tag=${ha_nonmpi_load_tags[$load]} - local cmd=${ha_nonmpi_load_cmds[$load]} - local dir=$ha_test_dir/$client-$tag - local log=$ha_tmp_dir/$client-$tag - local rc=0 - local nr_loops=0 - local start_time=$(date +%s) + local client=$1 + local load=$2 + local status=$3 + local tag=${ha_nonmpi_load_tags[$load]} + local cmd=${ha_nonmpi_load_cmds[$tag]} + local test_dir=$4 + local dir=$test_dir/$client-$tag + local log=$ha_tmp_dir/$client-$tag + local rc=0 - cmd=${cmd//"{}"/$dir} + local rccheck=0 + local nr_loops=0 + local avg_loop_time=0 + local start_time=$(date +%s) + local check_attrs=${ha_check_attrs//"{}"/$dir} + + cmd=${cmd//"{}"/$dir} - ha_info "Starting $tag on $client" + ha_info "Starting $tag on $client on $dir" while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do + ha_info "$client Starts: $cmd" 2>&1 | tee -a $log ha_on $client "mkdir -p $dir && \ - $cmd && \ - rm -rf $dir" >>"$log" 2>&1 || rc=$? + $cmd" >>"$log" 2>&1 || rc=$? - if ((rc != 0)); then + ha_on $client "$check_attrs && \ + $LFS df $dir && \ + $check_attrs " >>"$log" 2>&1 && rccheck=1 || + ha_on $client "rm -rf $dir" >>"$log" 2>&1 + + ha_info rc=$rc rccheck=$rccheck + + if (( (rc + rccheck) != 0 )); then ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" - touch "$ha_fail_file" - touch "$ha_stop_file" + ha_touch stop,fail $client,$tag fi echo $rc >"$status" nr_loops=$((nr_loops + 1)) done - avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) + [ $nr_loops -ne 0 ] && + avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) - ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s" + ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s" } ha_start_nonmpi_loads() { - local client - local load - local tag - local status - - for client in ${ha_clients[@]}; do - for ((load = 0; load < ${#ha_nonmpi_load_tags[@]}; load++)); do - tag=${ha_nonmpi_load_tags[$load]} - status=$ha_status_file_prefix-$tag-$client - ha_repeat_nonmpi_load $client $load $status & - ha_status_files+=("$status") - done - done + local client + local load + local tag + local status + local n + local test_dir + local ndir + + for (( n = 0; n < ${#ha_clients[@]}; n++)); do + client=${ha_clients[n]} + ndir=$((n % ${#ha_testdirs[@]})) + test_dir=${ha_testdirs[ndir]} + for ((load = 0; load < ${#ha_nonmpi_load_tags[@]}; load++)); do + tag=${ha_nonmpi_load_tags[$load]} + status=$ha_status_file_prefix-$tag-$client + ha_repeat_nonmpi_load $client $load $status $test_dir & + ha_status_files+=("$status") + done + done +} + +declare ha_bgcmd=${ha_bgcmd:-""} +declare ha_bgcmd_log=$ha_tmp_dir/bgcmdlog + +ha_cmd_bg () { + [[ -z "$ha_bgcmd" ]] && return 0 + for ((i=0; i<${#ha_testdirs[@]}; i++)); do + ha_bgcmd=${ha_bgcmd//"{}"/${ha_testdirs[i]}} + done + + ha_info "BG cmd: $ha_bgcmd" + while [ true ]; do + [ -f $ha_stop_file ] && + ha_info "$ha_stop_file found! $ha_bgcmd no started" && + break + eval $ha_bgcmd 2>&1 | tee -a $ha_bgcmd_log + sleep 1 + done & + CMD_BG_PID=$! + ha_info CMD BG PID: $CMD_BG_PID + ps aux | grep $CMD_BG_PID +} + +ha_lfsck_bg () { + rm -f $ha_lfsck_log + rm -f $ha_lfsck_stop + + ha_info "LFSCK BG" + while [ true ]; do + [ -f $ha_lfsck_stop ] && ha_info "LFSCK stopped" && break + [ -f $ha_stop_file ] && + ha_info "$ha_stop_file found! LFSCK not started" && + break + ha_start_lfsck 2>&1 | tee -a $ha_lfsck_log + sleep 1 + done & + LFSCK_BG_PID=$! + ha_info LFSCK BG PID: $LFSCK_BG_PID +} + +ha_wait_lfsck_completed () { + local -a status + local -a types=($ha_lfsck_types) + local type + local s + + local nodes="${ha_servers[@]}" + nodes=${nodes// /,} + + # -A start LFSCK on all nodes + # -t default all + [ ${#types[@]} -eq 0 ] && types=(namespace layout) + ha_info "Waiting LFSCK completed in $ha_lfsck_wait sec: types ${types[@]}" + for type in ${types[@]}; do + eval var_$type=0 + for (( i=0; i<=ha_lfsck_wait; i++)); do + status=($(ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null | \ + awk '/status/ { print $3 }')) + for (( s=0; s<${#status[@]}; s++ )); do + # "partial" is expected after HARD failover + [[ "${status[s]}" = "completed" ]] || + [[ "${status[s]}" = "partial" ]] || break + done + [[ $s -eq ${#status[@]} ]] && eval var_$type=1 && break + sleep 1 + done + ha_info "LFSCK $type status in $i sec:" + ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null | grep status + + done + + for type in ${types[@]}; do + local var=var_$type + ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null + [[ ${!var} -eq 1 ]] || + { ha_info "lfsck not completed in $ha_lfsck_wait sec"; + return 1; } + done + return 0 +} + +ha_start_lfsck() +{ + local -a types=($ha_lfsck_types) + local rc=0 + + # -A: start LFSCK on all nodes via the specified MDT device + # (see "-M" option) by single LFSCK command + local params=" -A -r $ha_lfsck_custom_params" + + # use specified device if set + [ -n "$ha_lfsck_device" ] && params="-M $ha_lfsck_device $params" + + # -t: check type(s) to be performed (default all) + # check only specified types if set + if [ ${#types[@]} -ne 0 ]; then + local type="${types[@]}" + params="$params -t ${type// /,}" + fi + + ha_info "LFSCK start $params" + ha_on $ha_lfsck_node "lctl lfsck_start $params" || rc=1 + if [ $rc -ne 0 ]; then + if [ -e $ha_lfsck_lock ]; then + rc=0 + ha_wait_unlock $ha_lfsck_lock + ha_sleep 120 + ha_on $ha_lfsck_node "lctl lfsck_start $params" || rc=1 + fi + fi + + [ $rc -eq 0 ] || + { ha_touch stop,fail,lfsck; return 1; } + + ha_wait_lfsck_completed || + { ha_touch stop,fail,lfsck; return 1; } + + return 0 +} + +ha_lfsck_repaired() +{ + local n=0 + + n=$(cat $ha_lfsck_log | awk '/repaired/ {print $3}' |\ + awk '{sum += $1} END { print sum }') + [ $n -eq 0] || + { ha_info "Total repaired: $n"; + ha_touch fail; return 1; } + return 0 } ha_start_loads() { - trap ha_trap_stop_signals $ha_stop_signals - ha_start_nonmpi_loads - ha_start_mpi_loads + ha_cmd_bg + $ha_lfsck_bg && ha_lfsck_bg + trap ha_trap_stop_signals $ha_stop_signals + ha_start_nonmpi_loads + ha_start_mpi_loads } ha_stop_loads() { - touch $ha_stop_file - trap - $ha_stop_signals - ha_info "Waiting for workloads to stop" - wait + ha_touch stop + [[ -n $CMD_BG_PID ]] && wait $CMD_BG_PID || true + # true because of lfsck_bg could be stopped already + $ha_lfsck_bg && wait $LFSCK_BG_PID || true + trap - $ha_stop_signals + ha_info "Waiting for workloads to stop" + wait } ha_wait_loads() @@ -520,7 +866,7 @@ ha_wait_loads() local file local end=$(($(date +%s) + ha_load_timeout)) - ha_info "Waiting for workload status" + ha_info "Waiting $ha_load_timeout sec for workload status..." rm -f "${ha_status_files[@]}" # @@ -554,20 +900,181 @@ ha_wait_loads() done } +ha_powermanage() +{ + local nodes=$1 + local expected_state=$2 + local state + local -a states + local i + local rc=0 + + # store pm -x -q $nodes results in a file to have + # more information about nodes statuses + ha_on $ha_pm_host pm -x -q $nodes | awk '{print $2 $3}' > $ha_pm_states + rc=${PIPESTATUS[0]} + echo pmrc=$rc + + while IFS=": " read node state; do + [[ "$state" = "$expected_state" ]] && { + nodes=${nodes/$node/} + nodes=${nodes//,,/,} + nodes=${nodes/#,} + nodes=${nodes/%,} + } + done < $ha_pm_states + + if [ -n "$nodes" ]; then + cat $ha_pm_states + return 1 + fi + return 0 +} + +ha_power_down_cmd_fn() +{ + local nodes=$1 + local cmd + local pid + local rc=0 + + case $ha_power_down_cmd in + # format is: POWER_DOWN=sysrqcrash + sysrqcrash) + cmd="pdsh -S -w $nodes -u 120 \"echo c > /proc/sysrq-trigger\" &" + eval $cmd + pid=$! + ha_power_down_pids=$(echo $ha_power_down_pids $pid) + ha_info "ha_power_down_pids: $ha_power_down_pids" + [[ -z "$ha_power_down_pids" ]] || + ps aux | grep " ${ha_power_down_pids// / \| } " || + true + ;; + *) + cmd="$ha_power_down_cmd $nodes" + eval $cmd + rc=$? + ;; + esac + + return $rc +} + ha_power_down() { - local node=$1 + local nodes=$1 + local rc=1 + local i + local state + + case $ha_power_down_cmd in + *pm*) state=off ;; + sysrqcrash) state=off ;; + *) state=on;; + esac + + if $ha_lfsck_bg && [[ ${nodes//,/ /} =~ $ha_lfsck_node ]]; then + ha_info "$ha_lfsck_node down, delay start LFSCK" + ha_lock $ha_lfsck_lock + fi + + ha_info "Powering down $nodes : cmd: $ha_power_down_cmd" + ha_power_down_pids="" + for (( i=0; i<10; i++ )) { + ha_info "attempt: $i" + ha_power_down_cmd_fn $nodes || rc=1 + ha_sleep $ha_power_delay + ha_powermanage $nodes $state && rc=0 && break + } + if [[ -n "$ha_power_down_pids" ]]; then + kill -9 $ha_power_down_pids || true + wait $ha_power_down_pids || true + fi + + [ $rc -eq 0 ] || { + ha_info "Failed Powering down in $i attempts:" \ + "$ha_power_down_cmd" + cat $ha_pm_states + exit 1 + } +} + +ha_get_pair() +{ + local node=$1 + local i + + for ((i=0; i<${#ha_victims[@]}; i++)) { + [[ ${ha_victims[i]} == $node ]] && echo ${ha_victims_pair[i]} && + return + } + [[ $i -ne ${#ha_victims[@]} ]] || + ha_error "No pair found!" +} + +ha_power_up_delay() +{ + local nodes=$1 + local end=$(($(date +%s) + ha_node_up_delay)) + local rc + + if [[ ${#ha_victims_pair[@]} -eq 0 ]]; then + ha_sleep $ha_node_up_delay + return 0 + fi + + # Check CRM status on failover pair + while (($(date +%s) <= end)); do + rc=0 + for n in ${nodes//,/ }; do + local pair=$(ha_get_pair $n) + local status=$(ha_on $pair crm_mon -1rQ | \ + grep -w $n | head -1) + + ha_info "$n pair: $pair status: $status" + [[ "$status" == *OFFLINE* ]] || + rc=$((rc + $?)) + ha_info "rc: $rc" + done + + if [[ $rc -eq 0 ]]; then + ha_info "CRM: Got all victims status OFFLINE" + return 0 + fi + sleep 60 + done + + ha_info "$nodes CRM status not OFFLINE" + for n in ${nodes//,/ }; do + local pair=$(ha_get_pair $n) - ha_info "Powering down $node" - $ha_power_down_cmd $node + ha_info "CRM --- $n" + ha_on $pair crm_mon -1rQ + done + ha_error "CRM: some of $nodes are not OFFLINE in $ha_node_up_delay sec" + exit 1 } ha_power_up() { - local node=$1 + local nodes=$1 + local rc=1 + local i + + ha_power_up_delay $nodes + ha_info "Powering up $nodes : cmd: $ha_power_up_cmd" + for (( i=0; i<10; i++ )) { + ha_info "attempt: $i" + $ha_power_up_cmd $nodes && + ha_powermanage $nodes on && rc=0 && break + sleep $ha_power_delay + } - ha_info "Powering up $node" - $ha_power_up_cmd $node + [ $rc -eq 0 ] || { + ha_info "Failed Powering up in $i attempts: $ha_power_up_cmd" + cat $ha_pm_states + exit 1 + } } # @@ -587,28 +1094,52 @@ ha_rand() ha_aim() { - local i=$(ha_rand ${#ha_victims[@]}) + local i + local nodes - echo -n ${ha_victims[$i]} + if $ha_simultaneous ; then + nodes=$(echo ${ha_victims[@]}) + nodes=${nodes// /,} + else + i=$(ha_rand ${#ha_victims[@]}) + nodes=${ha_victims[$i]} + fi + + echo -n $nodes } -ha_wait_node() +ha_wait_nodes() { - local node=$1 - local end=$(($(date +%s) + 10 * 60)) + local nodes=$1 + local end=$(($(date +%s) + $ha_wait_nodes_up)) - ha_info "Waiting for $node to boot up" - until ha_on $node hostname >/dev/null 2>&1 || + ha_info "Waiting for $nodes to boot up in $ha_wait_nodes_up" + until ha_on $nodes hostname >/dev/null 2>&1 || [ -e "$ha_stop_file" ] || (($(date +%s) >= end)); do ha_sleep 1 >/dev/null done + + ha_info "Check where we are ..." + [ -e "$ha_stop_file" ] && + ha_info "$ha_stop_file found!" + + local -a nodes_up + nodes_up=($(ha_on $nodes hostname | awk '{ print $2 }')) + ha_info "Nodes $nodes are up: ${nodes_up[@]}" + local -a n=(${nodes//,/ }) + if [[ ${#nodes_up[@]} -ne ${#n[@]} ]]; then + ha_info "Failed boot up $nodes in $ha_wait_nodes_up sec!" + ha_touch fail,stop + return 1 + fi + return 0 } ha_failback() { - local node=$1 - ha_info "Failback resources on $node in $ha_failback_delay sec" + local nodes=$1 + ha_info "Failback resources on $nodes in $ha_failback_delay sec" ha_sleep $ha_failback_delay [ "$ha_failback_cmd" ] || @@ -617,7 +1148,8 @@ ha_failback() return 0 } - $ha_failback_cmd $node + $ha_failback_cmd $nodes + [ -e $ha_lfsck_lock ] && ha_unlock $ha_lfsck_lock || true } ha_summarize() @@ -630,34 +1162,34 @@ ha_summarize() ha_killer() { - local node + local nodes while (($(date +%s) < ha_start_time + ha_expected_duration)) && [ ! -e "$ha_stop_file" ]; do ha_info "---------------8<---------------" - $ha_workloads_only || node=$(ha_aim) + $ha_workloads_only || nodes=$(ha_aim) - ha_info "Failing $node" + ha_info "Failing $nodes" $ha_workloads_only && ha_info " is skipped: workload only..." ha_sleep $(ha_rand $ha_max_failover_period) - $ha_workloads_only || ha_power_down $node + $ha_workloads_only || ha_power_down $nodes ha_sleep 10 ha_wait_loads || return if [ -e $ha_stop_file ]; then - $ha_workloads_only || ha_power_up $node + $ha_workloads_only || ha_power_up $nodes break fi - ha_info "Bringing $node back" + ha_info "Bringing $nodes back" ha_sleep $(ha_rand 10) $ha_workloads_only || { - ha_power_up $node - ha_wait_node $node - ha_failback $node + ha_power_up $nodes + ha_wait_nodes $nodes + ha_failback $nodes } # @@ -683,9 +1215,25 @@ ha_main() "START: $0: $(date +%H:%M:%S' '%s)" trap ha_trap_exit EXIT mkdir "$ha_tmp_dir" - ha_on ${ha_clients[0]} mkdir "$ha_test_dir" - ha_on ${ha_clients[0]} " \ - $LFS setstripe $ha_stripe_params $ha_test_dir" + + local mdt_index + if $ha_test_dir_mdt_index_random && + [ $ha_test_dir_mdt_index -ne 0 ]; then + mdt_index=$(ha_rand $((ha_test_dir_mdt_index + 1)) ) + else + mdt_index=$ha_test_dir_mdt_index + fi + + local dir + test_dir=${ha_testdirs[0]} + ha_on ${ha_clients[0]} "$LFS mkdir -i$mdt_index \ + -c$ha_test_dir_stripe_count $test_dir" + for ((i=0; i<${#ha_testdirs[@]}; i++)); do + test_dir=${ha_testdirs[i]} + ha_on ${ha_clients[0]} $LFS getdirstripe $test_dir + ha_on ${ha_clients[0]} " \ + $LFS setstripe $ha_stripe_params $test_dir" + done ha_start_loads ha_wait_loads @@ -699,6 +1247,10 @@ ha_main() ha_stop_loads + $ha_lfsck_after && ha_start_lfsck | tee -a $ha_lfsck_log + + $ha_lfsck_fail_on_repaired && ha_lfsck_repaired + if [ -e "$ha_fail_file" ]; then exit 1 else