X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fha.sh;h=987387b1c8efb018cdc2a7525b809c2572488787;hb=050c2fb23b1f98745305a3dfe3062ea5a66dfdb4;hp=c26689d8c73559b7f7173d0c1e295b3541a6001a;hpb=32e96e1a48524da7aa5d2aebc8a217549d763f75;p=fs%2Flustre-release.git diff --git a/lustre/tests/ha.sh b/lustre/tests/ha.sh index c26689d..987387b 100755 --- a/lustre/tests/ha.sh +++ b/lustre/tests/ha.sh @@ -144,6 +144,21 @@ ha_info() echo "$0: $(date +%H:%M:%S' '%s):" "$@" } +ha_touch() +{ + local date=$(date +%H:%M:%S' '%s) + + [[ $1 =~ stop ]] && + echo $date ${FUNCNAME[1]} $2 >> $ha_stop_file || + true + [[ $1 =~ fail ]] && + echo $date ${FUNCNAME[1]} $2 >> $ha_fail_file || + true + [[ $1 =~ lfsck ]] && + echo $date ${FUNCNAME[1]} $2 >> $ha_lfsck_stop || + true +} + ha_log() { local nodes=${1// /,} @@ -171,6 +186,7 @@ ha_trap_err() trap ha_trap_err ERR set -eE +declare ha_power_down_pids declare ha_tmp_dir=/tmp/$(basename $0)-$$ declare ha_stop_file=$ha_tmp_dir/stop declare ha_fail_file=$ha_tmp_dir/fail @@ -193,11 +209,16 @@ declare ha_power_down_cmd=${POWER_DOWN:-"pm -0"} declare ha_power_up_cmd=${POWER_UP:-"pm -1"} declare ha_power_delay=${POWER_DELAY:-60} declare ha_node_up_delay=${NODE_UP_DELAY:-10} +declare ha_wait_nodes_up=${WAIT_NODES_UP:-600} declare ha_pm_host=${PM_HOST:-$(hostname)} declare ha_failback_delay=${DELAY:-5} declare ha_failback_cmd=${FAILBACK:-""} declare ha_stripe_params=${STRIPEPARAMS:-"-c 0"} +declare ha_test_dir_stripe_count=${TDSTRIPECOUNT:-"1"} +declare ha_test_dir_mdt_index=${TDMDTINDEX:-"0"} +declare ha_test_dir_mdt_index_random=${TDMDTINDEXRAND:-false} declare ha_dir_stripe_count=${DSTRIPECOUNT:-"1"} +declare ha_dir_stripe_count_random=${DSTRIPECOUNTRAND:-false} declare ha_mdt_index=${MDTINDEX:-"0"} declare ha_mdt_index_random=${MDTINDEXRAND:-false} declare -a ha_clients @@ -205,6 +226,15 @@ declare -a ha_servers declare -a ha_victims declare -a ha_victims_pair declare ha_test_dir=/mnt/lustre/$(basename $0)-$$ +declare -a ha_testdirs=(${ha_test_dirs="$ha_test_dir"}) + +for ((i=0; i<${#ha_testdirs[@]}; i++)); do + echo I=$i ${ha_testdirs[i]} + ha_testdirs[i]="${ha_testdirs[i]}/$(basename $0)-$$" + echo i=$i ${ha_testdirs[i]} +done + +declare ha_cleanup=${CLEANUP:-true} declare ha_start_time=$(date +%s) declare ha_expected_duration=$((60 * 60 * 24)) declare ha_max_failover_period=10 @@ -220,6 +250,18 @@ declare ha_mpi_instances=${ha_mpi_instances:-1} declare ha_mpi_loads=${ha_mpi_loads="ior simul mdtest"} declare -a ha_mpi_load_tags=($ha_mpi_loads) declare -a ha_mpiusers=(${ha_mpi_users="mpiuser"}) +declare -a ha_users +declare -A ha_mpiopts + +for ((i=0; i<${#ha_mpiusers[@]}; i++)); do + u=${ha_mpiusers[i]%%:*} + o="" + # user gets empty option if ha_mpi_users does not specify it explicitly + [[ ${ha_mpiusers[i]} =~ : ]] && o=${ha_mpiusers[i]##*:} + ha_users[i]=$u + ha_mpiopts[$u]+=" $o" +done +ha_users=(${!ha_mpiopts[@]}) declare ha_ior_params=${IORP:-'" -b $ior_blockSize -t 2m -w -W -T 1"'} declare ha_simul_params=${SIMULP:-'" -n 10"'} @@ -257,6 +299,7 @@ declare -A ha_nonmpi_load_cmds=( [iozone]="iozone -a -e -+d -s $iozone_SIZE {}/f.iozone" [racer]="$ha_racer_params $racer {}" ) +declare ha_check_attrs="find {} -type f -ls 2>&1 | grep -e '?'" ha_usage() { @@ -337,29 +380,31 @@ ha_on() # -S is to be used here to track the # remote command return values # - pdsh -S -w $nodes PATH=/usr/local/sbin:/usr/local/bin:/sbin:\ -/bin:/usr/sbin:/usr/bin "$@" || + pdsh -S -w $nodes "PATH=/usr/local/sbin:/usr/local/bin:/sbin:\ +/bin:/usr/sbin:/usr/bin; $@" || rc=$? return $rc } ha_trap_exit() { - touch "$ha_stop_file" + ha_touch stop trap 0 if [ -e "$ha_fail_file" ]; then - ha_info "Test directory $ha_test_dir not removed" + ha_info "Test directories ${ha_testdirs[@]} not removed" ha_info "Temporary directory $ha_tmp_dir not removed" else - ha_on ${ha_clients[0]} rm -rf "$ha_test_dir" + $ha_cleanup && + ha_on ${ha_clients[0]} rm -rf ${ha_testdirs[@]} || + ha_info "Test directories ${ha_testdirs[@]} not removed" ha_info "Please find the results in the directory $ha_tmp_dir" fi } ha_trap_stop_signals() { - ha_info "${ha_stop_signals// /,} received" - touch "$ha_stop_file" + ha_info "${ha_stop_signals// /,} received" + ha_touch stop "${ha_stop_signals// /,} received" } ha_sleep() @@ -429,64 +474,88 @@ ha_repeat_mpi_load() local stripeparams=$6 local mpiuser=$7 local mustpass=$8 + local mpirunoptions=$9 + local test_dir=${10} local tag=${ha_mpi_load_tags[$load]} local cmd=${ha_mpi_load_cmds[$tag]} - local dir=$ha_test_dir/$client-$tag + local dir=$test_dir/$client-$tag local log=$ha_tmp_dir/$client-$tag local rc=0 + local rccheck=0 local nr_loops=0 local avg_loop_time=0 local start_time=$(date +%s) + local check_attrs=${ha_check_attrs//"{}"/$dir} cmd=${cmd//"{}"/$dir} cmd=${cmd//"{params}"/$parameter} [[ -n "$ha_postcmd" ]] && ha_postcmd=${ha_postcmd//"{}"/$dir} + [[ -n "$ha_precmd" ]] && ha_precmd=${ha_precmd//"{}"/$dir} ha_info "Starting $tag" machines="-machinefile $machines" - while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do - ha_info "$client Starts: $cmd" 2>&1 | tee -a $log + while [ ! -e "$ha_stop_file" ] && ((rc == 0)) && ((rccheck == 0)); do + ha_info "$client Starts: $mpiuser: $cmd" 2>&1 | tee -a $log { local mdt_index if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then - mdt_index=$(ha_rand $ha_mdt_index) + mdt_index=$(ha_rand $((ha_mdt_index + 1)) ) else mdt_index=$ha_mdt_index fi - ha_on $client $LFS mkdir -i$mdt_index -c$ha_dir_stripe_count "$dir" && + local dir_stripe_count + if $ha_dir_stripe_count_random && + [ $ha_dir_stripe_count -ne 1 ]; then + dir_stripe_count=$(($(ha_rand $ha_dir_stripe_count) + 1)) + else + dir_stripe_count=$ha_dir_stripe_count + fi + [[ -n "$ha_precmd" ]] && ha_info "$ha_precmd" && + ha_on $client "$ha_precmd" >>"$log" 2>&1 + ha_info "$client Creates $dir with -i$mdt_index -c$dir_stripe_count " + ha_on $client $LFS mkdir -i$mdt_index -c$dir_stripe_count "$dir" && ha_on $client $LFS getdirstripe "$dir" && ha_on $client $LFS setstripe $stripeparams $dir && ha_on $client $LFS getstripe $dir && ha_on $client chmod a+xwr $dir && - ha_on $client "su $mpiuser sh -c \" $mpirun $ha_mpirun_options \ - -np $((${#ha_clients[@]} * mpi_threads_per_client )) \ + ha_on $client "su $mpiuser bash -c \" $mpirun $mpirunoptions \ + -np $((${#ha_clients[@]} * mpi_threads_per_client / ha_nclientsset)) \ $machines $cmd \" " || rc=$? + ha_on ${ha_clients[0]} "$check_attrs && \ + $LFS df $dir && \ + $check_attrs " && rccheck=1 [[ -n "$ha_postcmd" ]] && ha_info "$ha_postcmd" && - ha_on $client $ha_postcmd >>"$log" 2>&1 - (( ((rc == 0)) && (( mustpass != 0 )) )) || - (( ((rc != 0)) && (( mustpass == 0 )) )) && - ha_on $client rm -rf "$dir"; - } >>"$log" 2>&1 || rc=$? - - ha_info rc=$rc mustpass=$mustpass + ha_on $client "$ha_postcmd" >>"$log" 2>&1 + if (( ((rc == 0)) && ((rccheck == 0)) && \ + (( mustpass != 0 )) )) || + (( ((rc != 0)) && ((rccheck == 0)) && \ + (( mustpass == 0 )) )); then + local suf=$(date +%s) + $ha_cleanup && ha_on $client rm -rf "$dir" || + ha_on $client mv "$dir" "${dir}.${suf}" + fi; + } >>"$log" 2>&1 + + ha_info $client: rc=$rc rccheck=$rccheck mustpass=$mustpass # mustpass=0 means that failure is expected - if (( rc !=0 )); then + if (( rccheck != 0 )); then + ha_touch stop,fail $client,$tag + ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" + elif (( rc !=0 )); then if (( mustpass != 0 )); then - touch "$ha_fail_file" - touch "$ha_stop_file" + ha_touch stop,fail $client,$tag ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" else # Ok to fail rc=0 fi elif (( mustpass == 0 )); then - touch "$ha_fail_file" - touch "$ha_stop_file" + ha_touch stop,fail $client,$tag ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" fi - echo rc=$rc mustpass=$mustpass >"$status" + echo rc=$rc rccheck=$rccheck mustpass=$mustpass >"$status" nr_loops=$((nr_loops + 1)) done @@ -510,6 +579,7 @@ ha_start_mpi_loads() local m local -a mach local mpiuser + local nmpi # ha_mpi_instances defines the number of # clients start mpi loads; should be <= ${#ha_clients[@]} @@ -541,9 +611,13 @@ ha_start_mpi_loads() scp $ha_machine_file* $client:$dirname done + local ndir for ((n = 0; n < $inst; n++)); do client=${ha_clients[n]} - mpiuser=${ha_mpiusers[$((n % ${#ha_mpiusers[@]}))]} + nmpi=$((n % ${#ha_users[@]})) + mpiuser=${ha_users[nmpi]} + ndir=$((n % ${#ha_testdirs[@]})) + test_dir=${ha_testdirs[ndir]} for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do tag=${ha_mpi_load_tags[$load]} status=$ha_status_file_prefix-$tag-$client @@ -562,7 +636,8 @@ ha_start_mpi_loads() [[ $ha_ninstmustfail == 0 ]] || mustpass=$(( n % ha_ninstmustfail )) ha_repeat_mpi_load $client $load $status "$parameter" \ - $machines "$stripe" "$mpiuser" "$mustpass" & + $machines "$stripe" "$mpiuser" "$mustpass" \ + "${ha_mpiopts[$mpiuser]} $ha_mpirun_options" "$test_dir" & ha_status_files+=("$status") done done @@ -575,27 +650,36 @@ ha_repeat_nonmpi_load() local status=$3 local tag=${ha_nonmpi_load_tags[$load]} local cmd=${ha_nonmpi_load_cmds[$tag]} - local dir=$ha_test_dir/$client-$tag + local test_dir=$4 + local dir=$test_dir/$client-$tag local log=$ha_tmp_dir/$client-$tag local rc=0 + + local rccheck=0 local nr_loops=0 local avg_loop_time=0 local start_time=$(date +%s) + local check_attrs=${ha_check_attrs//"{}"/$dir} cmd=${cmd//"{}"/$dir} - ha_info "Starting $tag on $client" + ha_info "Starting $tag on $client on $dir" while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do ha_info "$client Starts: $cmd" 2>&1 | tee -a $log ha_on $client "mkdir -p $dir && \ - $cmd && \ - rm -rf $dir" >>"$log" 2>&1 || rc=$? + $cmd" >>"$log" 2>&1 || rc=$? + + ha_on $client "$check_attrs && \ + $LFS df $dir && \ + $check_attrs " >>"$log" 2>&1 && rccheck=1 || + ha_on $client "rm -rf $dir" >>"$log" 2>&1 - if ((rc != 0)); then + ha_info rc=$rc rccheck=$rccheck + + if (( (rc + rccheck) != 0 )); then ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}" - touch "$ha_fail_file" - touch "$ha_stop_file" + ha_touch stop,fail $client,$tag fi echo $rc >"$status" @@ -610,19 +694,47 @@ ha_repeat_nonmpi_load() ha_start_nonmpi_loads() { - local client - local load - local tag - local status - - for client in ${ha_clients[@]}; do - for ((load = 0; load < ${#ha_nonmpi_load_tags[@]}; load++)); do - tag=${ha_nonmpi_load_tags[$load]} - status=$ha_status_file_prefix-$tag-$client - ha_repeat_nonmpi_load $client $load $status & - ha_status_files+=("$status") - done - done + local client + local load + local tag + local status + local n + local test_dir + local ndir + + for (( n = 0; n < ${#ha_clients[@]}; n++)); do + client=${ha_clients[n]} + ndir=$((n % ${#ha_testdirs[@]})) + test_dir=${ha_testdirs[ndir]} + for ((load = 0; load < ${#ha_nonmpi_load_tags[@]}; load++)); do + tag=${ha_nonmpi_load_tags[$load]} + status=$ha_status_file_prefix-$tag-$client + ha_repeat_nonmpi_load $client $load $status $test_dir & + ha_status_files+=("$status") + done + done +} + +declare ha_bgcmd=${ha_bgcmd:-""} +declare ha_bgcmd_log=$ha_tmp_dir/bgcmdlog + +ha_cmd_bg () { + [[ -z "$ha_bgcmd" ]] && return 0 + for ((i=0; i<${#ha_testdirs[@]}; i++)); do + ha_bgcmd=${ha_bgcmd//"{}"/${ha_testdirs[i]}} + done + + ha_info "BG cmd: $ha_bgcmd" + while [ true ]; do + [ -f $ha_stop_file ] && + ha_info "$ha_stop_file found! $ha_bgcmd no started" && + break + eval $ha_bgcmd 2>&1 | tee -a $ha_bgcmd_log + sleep 1 + done & + CMD_BG_PID=$! + ha_info CMD BG PID: $CMD_BG_PID + ps aux | grep $CMD_BG_PID } ha_lfsck_bg () { @@ -714,12 +826,10 @@ ha_start_lfsck() fi [ $rc -eq 0 ] || - { touch "$ha_fail_file"; touch "$ha_stop_file"; - touch $ha_lfsck_stop; return 1; } + { ha_touch stop,fail,lfsck; return 1; } ha_wait_lfsck_completed || - { touch "$ha_fail_file"; touch "$ha_stop_file"; - touch $ha_lfsck_stop; return 1; } + { ha_touch stop,fail,lfsck; return 1; } return 0 } @@ -730,14 +840,15 @@ ha_lfsck_repaired() n=$(cat $ha_lfsck_log | awk '/repaired/ {print $3}' |\ awk '{sum += $1} END { print sum }') - [ $n -eq 0] || + (( n == 0 )) || { ha_info "Total repaired: $n"; - touch "$ha_fail_file"; return 1; } + ha_touch fail; return 1; } return 0 } ha_start_loads() { + ha_cmd_bg $ha_lfsck_bg && ha_lfsck_bg trap ha_trap_stop_signals $ha_stop_signals ha_start_nonmpi_loads @@ -746,7 +857,8 @@ ha_start_loads() ha_stop_loads() { - touch $ha_stop_file + ha_touch stop + [[ -n $CMD_BG_PID ]] && wait $CMD_BG_PID || true # true because of lfsck_bg could be stopped already $ha_lfsck_bg && wait $LFSCK_BG_PID || true trap - $ha_stop_signals @@ -828,14 +940,29 @@ ha_power_down_cmd_fn() { local nodes=$1 local cmd + local pid + local rc=0 case $ha_power_down_cmd in # format is: POWER_DOWN=sysrqcrash - sysrqcrash) cmd="pdsh -S -w $nodes 'echo c > /proc/sysrq-trigger' &" ;; - *) cmd="$ha_power_down_cmd $nodes" ;; + sysrqcrash) + cmd="pdsh -S -w $nodes -u 120 \"echo c > /proc/sysrq-trigger\" &" + eval $cmd + pid=$! + ha_power_down_pids=$(echo $ha_power_down_pids $pid) + ha_info "ha_power_down_pids: $ha_power_down_pids" + [[ -z "$ha_power_down_pids" ]] || + ps aux | grep " ${ha_power_down_pids// / \| } " || + true + ;; + *) + cmd="$ha_power_down_cmd $nodes" + eval $cmd + rc=$? + ;; esac - eval $cmd + return $rc } ha_power_down() @@ -857,12 +984,17 @@ ha_power_down() fi ha_info "Powering down $nodes : cmd: $ha_power_down_cmd" + ha_power_down_pids="" for (( i=0; i<10; i++ )) { ha_info "attempt: $i" - ha_power_down_cmd_fn $nodes && - ha_powermanage $nodes $state && rc=0 && break - sleep $ha_power_delay + ha_power_down_cmd_fn $nodes || rc=1 + ha_sleep $ha_power_delay + ha_powermanage $nodes $state && rc=0 && break } + if [[ -n "$ha_power_down_pids" ]]; then + kill -9 $ha_power_down_pids || true + wait $ha_power_down_pids || true + fi [ $rc -eq 0 ] || { ha_info "Failed Powering down in $i attempts:" \ @@ -984,14 +1116,29 @@ ha_aim() ha_wait_nodes() { local nodes=$1 - local end=$(($(date +%s) + 10 * 60)) + local end=$(($(date +%s) + $ha_wait_nodes_up)) - ha_info "Waiting for $nodes to boot up" + ha_info "Waiting for $nodes to boot up in $ha_wait_nodes_up" until ha_on $nodes hostname >/dev/null 2>&1 || [ -e "$ha_stop_file" ] || (($(date +%s) >= end)); do ha_sleep 1 >/dev/null done + + ha_info "Check where we are ..." + [ -e "$ha_stop_file" ] && + ha_info "$ha_stop_file found!" + + local -a nodes_up + nodes_up=($(ha_on $nodes hostname | awk '{ print $2 }')) + ha_info "Nodes $nodes are up: ${nodes_up[@]}" + local -a n=(${nodes//,/ }) + if [[ ${#nodes_up[@]} -ne ${#n[@]} ]]; then + ha_info "Failed boot up $nodes in $ha_wait_nodes_up sec!" + ha_touch fail,stop + return 1 + fi + return 0 } ha_failback() @@ -1073,9 +1220,25 @@ ha_main() "START: $0: $(date +%H:%M:%S' '%s)" trap ha_trap_exit EXIT mkdir "$ha_tmp_dir" - ha_on ${ha_clients[0]} mkdir "$ha_test_dir" - ha_on ${ha_clients[0]} " \ - $LFS setstripe $ha_stripe_params $ha_test_dir" + + local mdt_index + if $ha_test_dir_mdt_index_random && + [ $ha_test_dir_mdt_index -ne 0 ]; then + mdt_index=$(ha_rand $((ha_test_dir_mdt_index + 1)) ) + else + mdt_index=$ha_test_dir_mdt_index + fi + + local dir + test_dir=${ha_testdirs[0]} + ha_on ${ha_clients[0]} "$LFS mkdir -i$mdt_index \ + -c$ha_test_dir_stripe_count $test_dir" + for ((i=0; i<${#ha_testdirs[@]}; i++)); do + test_dir=${ha_testdirs[i]} + ha_on ${ha_clients[0]} $LFS getdirstripe $test_dir + ha_on ${ha_clients[0]} " \ + $LFS setstripe $ha_stripe_params $test_dir" + done ha_start_loads ha_wait_loads