X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre-iokit%2Fior-survey%2Fior-survey;h=a2d67245600ce5563d46d43488c4c9b2ac5422d8;hp=b7c7421e78675de916b02044f07671a7b2bbe252;hb=2e562ab4ad1c9b73c99ec7473a997a0e57ac8392;hpb=c1c331f718d4698b71e05df02892f9ef1286dd09 diff --git a/lustre-iokit/ior-survey/ior-survey b/lustre-iokit/ior-survey/ior-survey index b7c7421..a2d6724 100644 --- a/lustre-iokit/ior-survey/ior-survey +++ b/lustre-iokit/ior-survey/ior-survey @@ -1,63 +1,63 @@ #!/bin/bash +# the value of these can be set in the environment. +# This test assumes a typical pdsh naming scheme, where +# node names can be expressed as a string +# forllowed by a number # cluster name (all node names are this followed by the node number) -cluster=mdev +cluster=${cluster:-""} +if [ -z "$cluster" ];then echo "cluster not defined"; exit 1; fi # client node numbers (individual numbers or inclusive ranges) -clients=(7-8) +clients=${clients:-""} +if [ -z "$clients" ]; then echo "clients not defined"; exit 1; fi # numbers of clients to survey -clients_lo=1 -clients_hi=2 -clients_iterator="+=1" +clients_lo=${clients_lo:-1} +clients_hi=${clients_hi:-3} +clients_iterator=${clients_iterator:-"+=1"} # numbers of tasks per client to survey -tasks_per_client_lo=1 -tasks_per_client_hi=8 -tasks_per_client_iterator="*=2" +tasks_per_client_lo=${task_per_client_lo:-1} +tasks_per_client_hi=${tasks_per_client_hi:-8} +tasks_per_client_iterator=${tasks_per_client_iterator:-"*=2"} # record sizes to survey -rsize_lo=1M -rsize_hi=1M -rsize_iterator="*=2" +rsize_lo=${rsize_lo:-1M} +rsize_hi=${rsize_hi:-1M} +rsize_iterator=${rsize_iterator:-"*=2"} ## which tests to run (first must be write) -# remount) not really a test; just remount to uncache everything +# clear_cache) not really a test; just uncache everything # *write*) write # *) read -#tests=(write rewrite read reread rewrite_again) -tests=(write rewrite remount read reread) +tests=(write rewrite clear_cache read reread) # total # bytes written/read by any client node -min_per_client_size=4G -min_total_size=8G +min_per_client_size=${min_per_client_size:-"1G"} +min_total_size=${min_total_size:-"2G"} # should each task do I/O to its own file? -file_per_task=1 +file_per_task=${file_per_task:-1} # the binaries -IOR="/home/ericb/ior/src/C/IOR" -llmount=/home/ericb/lustre/utils/llmount -pdsh=pdsh +IOR=${IOR:-"/usr/local/sbin/IOR"} +llmount=${llmount:-"llmount"} +# Select mpirun, yod, or pdsh +fanout_cmd=${fanout_cmd:-"pdsh"} +# mpirun still uses pdsh for cleanup +pdsh=${pdsh:-"pdsh"} +pdsh_args="-R ssh -S -b -w " # the result file prefix (date/time + hostname makes unique) -#rslt=/home/ericb/ior_survey_`date +%F@%R`_`uname -n` -rslt=/home/ericb/ior_survey +rslt_loc=${rslt_loc:-"/tmp"} +rslt=${rslt:-"$rslt_loc/ior_survey_`date +%F@%R`_`uname -n`"} # where lustre is mounted on the clients -lustre=/mnt/lustre +lustre=${lustre:-"/mnt/lustre"} # basename of the test file(s) -testfile=${lustre}/ior_survey_testfile - -# how to unmount and remount the F/S on a client (to clear the cache) -# change this depending on lustre config (network type, MDS etc) -remount="umount $lustre && $llmount -o nettype=elan mdev6:/ll_mds/client $lustre" - -# pdsh args required to instantiate all instances of IOR in parallel -# the chosen module must support '-n ' -# -R, -f etc -pdsh_mpiargs="-Rmqsh" +testfile=${testfile:-"${lustre}/ior/ior_survey_testfile"} #don't spin for MPI completions export LIBELAN_WAITTYPE=0 @@ -65,6 +65,54 @@ export LIBELAN_WAITTYPE=0 ################################################################################ # dont change stuff below here unless you know what you're doing... +# This is to allow use of yod, pdsh, etc. +fanout() { + local clients=$1; shift + local tmpfile=$1; shift + local result + case $fanout_cmd in + 'pdsh') + $fanout_cmd $pdsh_args "$clients" "$@" >> $tmpfile 2>&1 + echo $? + return + ;; + 'mpirun') + # horrible misuse of globals + $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1 + echo $? + return + ;; + 'yod') + # and another + $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1 + echo $? + return + ;; + + *) + echo "255" + return + ;; + esac +} + +dump_cache() { + # we are assuming mpi uses will also have pdsh + local clients=$1;shift + local tmpfile=$1;shift + clear_cache='for LRU in /proc/fs/lustre/ldlm/namespaces/*/lru_size; + do sudo /bin/bash -c "echo clear > $LRU"; done' + echo "=> $clear_cache" >> $tmpfile + $pdsh $pdsh_args "$test_clients" "$clear_cache" >> $tmpfile 2>&1 + status=$? + echo "Completion Status: $status" >> $tmpfile + + if ((status)); then + echo "ERROR" + else + echo "OK" + fi +} count_range() { echo $1 | awk '{ nvals=split($1, vals, "-");\ if (nvals == 1) print 1;\ @@ -83,7 +131,7 @@ idx2nodenum() { return fi chunk=`count_range $range` - if ((chunk > n)); then + if ((chunk > $n)); then base=`base_range $range` echo $((base + n)) return @@ -103,7 +151,7 @@ n2noderange() { fi local base=`base_range $range` local chunk=`count_range $range` - if ((chunk > n)); then chunk=n; fi + if ((chunk > $n)); then chunk=$n; fi local nodes="${nodes}${sep}${base}"; sep="," if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi n=$((n-chunk)) @@ -139,11 +187,11 @@ pp_number() { local G=$((1024*1024*1024)) local M=$((1024*1024)) local K=$((1024)) - if ((n%G == 0 && n >= G)); then + if ((n%G == 0 && n >= $G)); then echo "$((n/G))G" - elif ((n%M == 0 && n >= M)); then + elif ((n%M == 0 && n >= $M)); then echo "$((n/M))M" - elif ((n%K == 0 && n >= K)); then + elif ((n%K == 0 && n >= $K)); then echo "$((n/K))K" else echo $n @@ -170,6 +218,21 @@ print_summary () { echo $minusn "$*" } +check_mount() { + local lustre=$1; shift + local tmpb=$1; shift + local clients=$1; shift + local tmpfile=${tmpb}_tmp + # check lustre is mounted everywhere it's needed + cmd="grep $lustre /proc/mounts" + $pdsh $pdsh_args "$clients" "$cmd" >> $tmpfile + status=$? + if (($status)); then + print_summary "Lustre NOT mounted on $lustre somewhere" + exit 1 + fi + +} # convert params to actual numbers min_per_client_size=`parse_number $min_per_client_size` min_total_size=`parse_number $min_total_size` @@ -179,20 +242,30 @@ rsize_hi=`parse_number $rsize_hi` # check on actual numbers of client nodes nclients=`countnodes ${clients[@]}` -if ((clients_hi > nclients)); then clients_hi=$nclients; fi +if ((clients_hi > $nclients)); then clients_hi=$nclients; fi + +cur_date=`date` +machine=`uname -n` +script_name=`echo $0 | cut -d "/" -f2` + +echo "$cur_date $script_name on $lustre from $machine" >> $workf -for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do +for ((rsize=$rsize_lo; rsize<=$rsize_hi; rsize$rsize_iterator)); do pp_rsize=`pp_number $rsize` - for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do + for ((nclnt=$clients_lo; nclnt<=$clients_hi; nclnt$clients_iterator)); do test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`" - + echo $test_clients + if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ];then + check_mount $lustre $workf $test_clients + fi per_client_size=$((min_total_size/nclnt)) - if ((per_client_size < min_per_client_size)); then + if ((per_client_size < $min_per_client_size)); then per_client_size=$min_per_client_size fi - for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do + for ((ntask=$tasks_per_client_lo; ntask <= $tasks_per_client_hi; \ + ntask$tasks_per_client_iterator)); do per_task_size=$((per_client_size/ntask)) if ((per_task_size%rsize != 0)); then per_task_size=$(((per_task_size/rsize + 1)*rsize)) @@ -211,32 +284,14 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do tmpf=${workf}_tmp echo -n > $tmpf - if [ "$test" = "remount" ]; then - echo "=> $remount" >> $tmpf - $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \ - "$remount" - status=$? - echo "Completion Status: $status" >> $tmpf - - if ((status)); then - result="ERROR" - else - result="OK" - fi + if [ "$test" = "clear_cache" ]; then + if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ]; then + result=`dump_cache $test_clients $tmpf` + else + echo "Haven't figured out how to clear cache" >> $tmpf + result="N/A" + fi else - # check lustre is mounted everywhere it's needed - cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre" - echo "=> Mount Check: $cmd" >> $tmpf - $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \ - "$cmd" - status=$? - echo "Completion Status: $status" >> $tmpf - if ((status)); then - cat $tmpf >> $workf - rm $tmpf - print_summary "Lustre NOT mounted on $lustre somewhere" - exit 1 - fi cmdline=( $IOR # the command @@ -250,13 +305,14 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do idx=${#cmdline[@]} # keep the test file(s) unless this is the last test - ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k" + #((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k" + cmdline[$((idx++))]="-k" # use the existing test file(s) unless this is the first test ((test_idx > 0)) && cmdline[$((idx++))]="-E" # file-per-task - ((file_per_task)) && cmdline[$((idx++))]="-F" + (($file_per_task)) && cmdline[$((idx++))]="-F" case "$test" in *write*) cmdline[$((idx++))]="-w" @@ -267,17 +323,21 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do echo "=> ${cmdline[@]}" >> $tmpf - $pdsh -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask >> $tmpf 2>&1 \ - "${cmdline[@]}" - status=$? + status=`fanout $test_clients $tmpf ${cmdline[@]}` echo "Completion Status: $status" >> $tmpf - if ((status)); then + if (($status)); then result="ERROR" else - result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\ + # pdsh adds an extra field + if [ "$fanout_cmd" = "pdsh" ]; then + result=`awk < $tmpf "/$awkstr/ {print $ 4; found=1; exit}\ + END {if (!found) print \"ERROR\"}"` + else + result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\ END {if (!found) print \"ERROR\"}"` + fi fi fi