X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fha.sh;h=347e3f34d51098407f1b9b4e59966ca4a617de47;hb=489afbe69d5b7a867e44f6c2364513e660cf862b;hp=bd47b7fedd0da9b80ea59c05ef7c13da39712a77;hpb=47ca85aeab63c3d29aeef4931e605c177b475447;p=fs%2Flustre-release.git diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh index bd47b7f..347e3f3 100755 --- a/lustre/tests/ha.sh +++ b/lustre/tests/ha.sh @@ -174,11 +174,26 @@ declare ha_fail_file=$ha_tmp_dir/fail declare ha_status_file_prefix=$ha_tmp_dir/status declare -a ha_status_files declare ha_machine_file=$ha_tmp_dir/machine_file +declare ha_lfsck_log=$ha_tmp_dir/lfsck.log +declare ha_lfsck_lock=$ha_tmp_dir/lfsck.lock +declare ha_lfsck_stop=$ha_tmp_dir/lfsck.stop +declare ha_lfsck_bg=${LFSCK_BG:-false} +declare ha_lfsck_after=${LFSCK_AFTER:-false} +declare ha_lfsck_node=${LFSCK_NODE:-""} +declare ha_lfsck_device=${LFSCK_DEV:-""} +declare ha_lfsck_types=${LFSCK_TYPES:-"namespace layout"} +declare ha_lfsck_custom_params=${LFSCK_CUSTOM_PARAMS:-""} +declare ha_lfsck_wait=${LFSCK_WAIT:-1200} +declare ha_lfsck_fail_on_repaired=${LFSCK_FAIL_ON_REPAIRED:-false} declare ha_power_down_cmd=${POWER_DOWN:-"pm -0"} declare ha_power_up_cmd=${POWER_UP:-"pm -1"} +declare ha_power_delay=${POWER_DELAY:-60} declare ha_failback_delay=${DELAY:-5} declare ha_failback_cmd=${FAILBACK:-""} declare ha_stripe_params=${STRIPEPARAMS:-"-c 0"} +declare ha_dir_stripe_count=${DSTRIPECOUNT:-"1"} +declare ha_mdt_index=${MDTINDEX:-"0"} +declare ha_mdt_index_random=${MDTINDEXRAND:-false} declare -a ha_clients declare -a ha_servers declare -a ha_victims @@ -202,14 +217,20 @@ declare ha_ior_params=${IORP:-'" -b $ior_blockSize -t 2m -w -W -T 1"'} declare ha_simul_params=${SIMULP:-'" -n 10"'} declare ha_mdtest_params=${MDTESTP:-'" -i 1 -n 1000"'} declare ha_mpirun_options=${MPIRUN_OPTIONS:-""} +declare ha_clients_stripe=${CLIENTSSTRIPE:-'"$STRIPEPARAMS"'} +declare ha_nclientsset=${NCLIENTSSET:-1} + +declare ha_racer_params=${RACERP:-"MDSCOUNT=1"} eval ha_params_ior=($ha_ior_params) eval ha_params_simul=($ha_simul_params) eval ha_params_mdtest=($ha_mdtest_params) +eval ha_stripe_clients=($ha_clients_stripe) declare ha_nparams_ior=${#ha_params_ior[@]} declare ha_nparams_simul=${#ha_params_simul[@]} declare ha_nparams_mdtest=${#ha_params_mdtest[@]} +declare ha_nstripe_clients=${#ha_stripe_clients[@]} declare -A ha_mpi_load_cmds=( [ior]="$IOR -o {}/f.ior {params}" @@ -217,12 +238,15 @@ declare -A ha_mpi_load_cmds=( [mdtest]="$MDTEST {params} -d {}" ) +declare racer=${RACER:-"$(dirname $0)/racer/racer.sh"} + declare ha_nonmpi_loads=${ha_nonmpi_loads="dd tar iozone"} declare -a ha_nonmpi_load_tags=($ha_nonmpi_loads) -declare -a ha_nonmpi_load_cmds=( - "dd if=/dev/zero of={}/f.dd bs=1M count=256" - "tar cf - /etc | tar xf - -C {}" - "iozone -a -e -+d -s $iozone_SIZE {}/f.iozone" +declare -A ha_nonmpi_load_cmds=( + [dd]="dd if=/dev/zero of={}/f.dd bs=1M count=256" + [tar]="tar cf - /etc | tar xf - -C {}" + [iozone]="iozone -a -e -+d -s $iozone_SIZE {}/f.iozone" + [racer]="$ha_racer_params $racer {}" ) ha_usage() @@ -337,6 +361,15 @@ ha_sleep() sleep $n || true } +ha_wait_unlock() +{ + local lock=$1 + + while [ -e $lock ]; do + sleep 1 + done +} + ha_lock() { local lock=$1 @@ -355,18 +388,21 @@ ha_unlock() ha_dump_logs() { - local nodes=${1// /,} - local file=/tmp/$(basename $0)-$$-$(date +%s).dk - local lock=$ha_tmp_dir/lock-dump-logs + local nodes=${1// /,} + local file=/tmp/$(basename $0)-$$-$(date +%s).dk + local lock=$ha_tmp_dir/lock-dump-logs + local rc=0 - ha_lock "$lock" - ha_info "Dumping lctl log to $file" + ha_lock "$lock" + ha_info "Dumping lctl log to $file" # # some nodes could crash, so # do not exit with error if not all logs are dumped # - ha_on $nodes "lctl dk >$file" || + ha_on $nodes "lctl dk >>$file" || rc=$? + + [ $rc -eq 0 ] || ha_error "not all logs are dumped! Some nodes are unreachable." ha_unlock "$lock" } @@ -377,12 +413,15 @@ ha_repeat_mpi_load() local load=$2 local status=$3 local parameter=$4 + local machines=$5 + local stripeparams=$6 local tag=${ha_mpi_load_tags[$load]} local cmd=${ha_mpi_load_cmds[$tag]} local dir=$ha_test_dir/$client-$tag local log=$ha_tmp_dir/$client-$tag local rc=0 local nr_loops=0 + local avg_loop_time=0 local start_time=$(date +%s) cmd=${cmd//"{}"/$dir} @@ -390,10 +429,19 @@ ha_repeat_mpi_load() ha_info "Starting $tag" - local machines="-machinefile $ha_machine_file" + machines="-machinefile $machines" while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do { - ha_on $client mkdir -p "$dir" && + local mdt_index + if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then + mdt_index=$(ha_rand $ha_mdt_index) + else + mdt_index=$ha_mdt_index + fi + ha_on $client $LFS mkdir -i$mdt_index -c$ha_dir_stripe_count "$dir" && + ha_on $client $LFS getdirstripe "$dir" && + ha_on $client $LFS setstripe $stripeparams $dir && + ha_on $client $LFS getstripe $dir && ha_on $client chmod a+xwr $dir && ha_on $client "su mpiuser sh -c \" $mpirun $ha_mpirun_options \ -np $((${#ha_clients[@]} * mpi_threads_per_client )) \ @@ -413,7 +461,8 @@ ha_repeat_mpi_load() nr_loops=$((nr_loops + 1)) done - avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) + [ $nr_loops -ne 0 ] && + avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) ha_info "$tag stopped: rc $rc avg loop time $avg_loop_time" } @@ -426,22 +475,40 @@ ha_start_mpi_loads() local status local n local nparam + local machines + local m + local -a mach - for client in ${ha_clients[@]}; do - ha_info ha_machine_file=$ha_machine_file - echo $client >> $ha_machine_file + # ha_mpi_instances defines the number of + # clients start mpi loads; should be <= ${#ha_clients[@]} + # do nothing if + # ha_mpi_instances = 0 + # or + # ${#ha_mpi_load_tags[@]} =0 + local inst=$ha_mpi_instances + (( inst == 0 )) || (( ${#ha_mpi_load_tags[@]} == 0 )) && + ha_info "no mpi load to start" && + return 0 + + (( inst <= ${#ha_clients[@]} )) || inst=${#ha_clients[@]} + + # Define names for machinefiles for each client set + for (( n=0; n < $ha_nclientsset; n++ )); do + mach[$n]=$ha_machine_file$n + done + + for ((n = 0; n < ${#ha_clients[@]}; n++)); do + m=$(( n % ha_nclientsset)) + machines=${mach[m]} + ha_info machine_file=$machines + echo ${ha_clients[n]} >> $machines done local dirname=$(dirname $ha_machine_file) for client in ${ha_clients[@]}; do ha_on $client mkdir -p $dirname - scp $ha_machine_file $client:$ha_machine_file + scp $ha_machine_file* $client:$dirname done - # ha_mpi_instances defines the number of - # clients start mpi loads; should be <= ${#ha_clients[@]} - local inst=$ha_mpi_instances - (( inst <= ${#ha_clients[@]} )) || inst=${#ha_clients[@]} - for ((n = 0; n < $inst; n++)); do client=${ha_clients[n]} for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do @@ -453,7 +520,12 @@ ha_start_mpi_loads() nparam=$((n % num)) local aref=ha_params_$tag[nparam] local parameter=${!aref} - ha_repeat_mpi_load $client $load $status "$parameter" & + local nstripe=$((n % ha_nstripe_clients)) + aref=ha_stripe_clients[nstripe] + local stripe=${!aref} + local m=$(( n % ha_nclientsset)) + machines=${mach[m]} + ha_repeat_mpi_load $client $load $status "$parameter" $machines "$stripe" & ha_status_files+=("$status") done done @@ -461,16 +533,17 @@ ha_start_mpi_loads() ha_repeat_nonmpi_load() { - local client=$1 - local load=$2 - local status=$3 - local tag=${ha_nonmpi_load_tags[$load]} - local cmd=${ha_nonmpi_load_cmds[$load]} - local dir=$ha_test_dir/$client-$tag - local log=$ha_tmp_dir/$client-$tag - local rc=0 - local nr_loops=0 - local start_time=$(date +%s) + local client=$1 + local load=$2 + local status=$3 + local tag=${ha_nonmpi_load_tags[$load]} + local cmd=${ha_nonmpi_load_cmds[$tag]} + local dir=$ha_test_dir/$client-$tag + local log=$ha_tmp_dir/$client-$tag + local rc=0 + local nr_loops=0 + local avg_loop_time=0 + local start_time=$(date +%s) cmd=${cmd//"{}"/$dir} @@ -491,9 +564,10 @@ ha_repeat_nonmpi_load() nr_loops=$((nr_loops + 1)) done - avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) + [ $nr_loops -ne 0 ] && + avg_loop_time=$((($(date +%s) - start_time) / nr_loops)) - ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s" + ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s" } ha_start_nonmpi_loads() @@ -513,19 +587,133 @@ ha_start_nonmpi_loads() done } +ha_lfsck_bg () { + rm -f $ha_lfsck_log + rm -f $ha_lfsck_stop + + ha_info "LFSCK BG" + while [ true ]; do + [ -f $ha_lfsck_stop ] && ha_info "LFSCK stopped" && break + [ -f $ha_stop_file ] && + ha_info "$ha_stop_file found! LFSCK not started" && + break + ha_start_lfsck 2>&1 | tee -a $ha_lfsck_log + sleep 1 + done & + LFSCK_BG_PID=$! + ha_info LFSCK BG PID: $LFSCK_BG_PID +} + +ha_wait_lfsck_completed () { + local -a status + local -a types=($ha_lfsck_types) + local type + local s + + local nodes="${ha_servers[@]}" + nodes=${nodes// /,} + + # -A start LFSCK on all nodes + # -t default all + [ ${#types[@]} -eq 0 ] && types=(namespace layout) + ha_info "Waiting LFSCK completed in $ha_lfsck_wait sec: types ${types[@]}" + for type in ${types[@]}; do + eval var_$type=0 + for (( i=0; i<=ha_lfsck_wait; i++)); do + status=($(ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null | \ + awk '/status/ { print $3 }')) + for (( s=0; s<${#status[@]}; s++ )); do + # "partial" is expected after HARD failover + [[ "${status[s]}" = "completed" ]] || + [[ "${status[s]}" = "partial" ]] || break + done + [[ $s -eq ${#status[@]} ]] && eval var_$type=1 && break + sleep 1 + done + ha_info "LFSCK $type status in $i sec:" + ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null | grep status + + done + + for type in ${types[@]}; do + local var=var_$type + ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null + [[ ${!var} -eq 1 ]] || + { ha_info "lfsck not completed in $ha_lfsck_wait sec"; + return 1; } + done + return 0 +} + +ha_start_lfsck() +{ + local -a types=($ha_lfsck_types) + local rc=0 + + # -A: start LFSCK on all nodes via the specified MDT device + # (see "-M" option) by single LFSCK command + local params=" -A -r $ha_lfsck_custom_params" + + # use specified device if set + [ -n "$ha_lfsck_device" ] && params="-M $ha_lfsck_device $params" + + # -t: check type(s) to be performed (default all) + # check only specified types if set + if [ ${#types[@]} -ne 0 ]; then + local type="${types[@]}" + params="$params -t ${type// /,}" + fi + + ha_info "LFSCK start $params" + ha_on $ha_lfsck_node "lctl lfsck_start $params" || rc=1 + if [ $rc -ne 0 ]; then + if [ -e $ha_lfsck_lock ]; then + rc=0 + ha_wait_unlock $ha_lfsck_lock + ha_sleep 120 + ha_on $ha_lfsck_node "lctl lfsck_start $params" || rc=1 + fi + fi + + [ $rc -eq 0 ] || + { touch "$ha_fail_file"; touch "$ha_stop_file"; + touch $ha_lfsck_stop; return 1; } + + ha_wait_lfsck_completed || + { touch "$ha_fail_file"; touch "$ha_stop_file"; + touch $ha_lfsck_stop; return 1; } + + return 0 +} + +ha_lfsck_repaired() +{ + local n=0 + + n=$(cat $ha_lfsck_log | awk '/repaired/ {print $3}' |\ + awk '{sum += $1} END { print sum }') + [ $n -eq 0] || + { ha_info "Total repaired: $n"; + touch "$ha_fail_file"; return 1; } + return 0 +} + ha_start_loads() { - trap ha_trap_stop_signals $ha_stop_signals - ha_start_nonmpi_loads - ha_start_mpi_loads + $ha_lfsck_bg && ha_lfsck_bg + trap ha_trap_stop_signals $ha_stop_signals + ha_start_nonmpi_loads + ha_start_mpi_loads } ha_stop_loads() { - touch $ha_stop_file - trap - $ha_stop_signals - ha_info "Waiting for workloads to stop" - wait + touch $ha_stop_file + # true because of lfsck_bg could be stopped already + $ha_lfsck_bg && wait $LFSCK_BG_PID || true + trap - $ha_stop_signals + ha_info "Waiting for workloads to stop" + wait } ha_wait_loads() @@ -570,17 +758,36 @@ ha_wait_loads() ha_power_down() { local nodes=$1 + local rc=1 + local i + + if $ha_lfsck_bg && [[ ${nodes//,/ /} =~ $ha_lfsck_node ]]; then + ha_info "$ha_lfsck_node down, delay start LFSCK" + ha_lock $ha_lfsck_lock + fi ha_info "Powering down $nodes" - $ha_power_down_cmd $nodes + for i in $(seq 1 5); do + $ha_power_down_cmd $nodes && rc=0 && break + sleep $ha_power_delay + done + + [ $rc -eq 0 ] || ha_info "Failed Powering down in $i attempts" } ha_power_up() { local nodes=$1 + local rc=1 + local i ha_info "Powering up $nodes" - $ha_power_up_cmd $nodes + for i in $(seq 1 5); do + $ha_power_up_cmd $nodes && rc=0 && break + sleep $ha_power_delay + done + + [ $rc -eq 0 ] || ha_info "Failed Powering up in $i attempts" } # @@ -640,6 +847,7 @@ ha_failback() } $ha_failback_cmd $nodes + [ -e $ha_lfsck_lock ] && ha_unlock $ha_lfsck_lock || true } ha_summarize() @@ -721,6 +929,10 @@ ha_main() ha_stop_loads + $ha_lfsck_after && ha_start_lfsck | tee -a $ha_lfsck_log + + $ha_lfsck_fail_on_repaired && ha_lfsck_repaired + if [ -e "$ha_fail_file" ]; then exit 1 else