3 # cluster name (all node names are this followed by the node number)
6 # client node numbers (individual numbers or inclusive ranges)
9 # numbers of clients to survey
12 clients_iterator="*=2"
14 # numbers of tasks per client to survey
17 tasks_per_client_iterator="*=2"
19 # record sizes to survey
24 # This line contains all of the possible tests.
25 # IMPORTANT:::if you want to remount, put it in your tests array
26 # in the order you want to remount to clear the cache. For ex:
27 # tests=(write rewrite remount read reread rewrite_again) has a
28 # remount between the rewrite and remount tests
31 # total # bytes written/read by any client node
32 min_per_client_size=1G
35 # should each task do I/O to its own file?
39 IOR="/spin/home/henken/IOR-2.8.4/src/C/IOR"
42 #Command to run IOR (pdsh,mpirun,yod)
44 #Path to binary for program specified in runior
45 pathtobin="$(which yod)"
46 #location of machines file for mpirun, this file
47 #will be built from the cluster and client ranges
51 # the result file prefix (date/time + hostname makes unique)
52 rslt=/spin/home/henken/ior_survey/ior_survey_`date +%F@%R`_`uname -n`
53 #rslt=/home/larry/ior_survey
55 # where lustre is mounted on the clients
56 lustre=/lustre/fs1/nic/
58 # basename of the test file(s)
59 testfile=${lustre}/ior_survey_testfile
62 # how to unmount and remount the F/S on a client (to clear the cache)
63 # change this depending on lustre config (network type, MDS etc)
64 unmount="umount $lustre"
65 remount="llmount pegasus:/mds1/client $lustre"
67 # pdsh args required to instantiate all instances of IOR in parallel
68 # the chosen module must support '-n <procs-per-node>'
69 # -R<module>, -f<fanout> etc
72 #don't spin for MPI completions
73 export LIBELAN_WAITTYPE=0
75 ################################################################################
76 # dont change stuff below here unless you know what you're doing...
79 echo $1 | awk '{ nvals=split($1, vals, "-");\
80 if (nvals == 1) print 1;\
81 else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
85 echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
92 if [ -z "$range" ]; then
95 chunk=`count_range $range`
96 if ((chunk > n)); then
97 base=`base_range $range`
110 local range=$1; shift
111 if [ -z "$range" ]; then
114 local base=`base_range $range`
115 local chunk=`count_range $range`
116 if ((chunk > n)); then chunk=n; fi
117 local nodes="${nodes}${sep}${base}"; sep=","
118 if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
127 while ((radix > 0)); do
128 local nodes=`n2noderange $((n+radix)) $@`
129 if [ -n "$nodes" ]; then
140 *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
141 *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
142 *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
149 local G=$((1024*1024*1024))
150 local M=$((1024*1024))
152 if ((n%G == 0 && n >= G)); then
154 elif ((n%M == 0 && n >= M)); then
156 elif ((n%K == 0 && n >= K)); then
163 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
164 echo "First test must be 'write'" 1>&2
168 rsltf="${rslt}.summary"
169 workf="${rslt}.detail"
174 if [ "$1" = "-n" ]; then
179 echo $minusn "$*" >> $rsltf
185 local base=`base_range $1`
187 local chunk=`count_range $1`
189 local high=$((base+chunk-1))
191 for ((nmpi=$base; nmpi<=$high; nmpi++)); do
192 echo $cluster$nmpi >> $machines
200 $pathtobin -np $((ntask*nclnt)) -machinefile $machines >> $tmpf 2>1 \
203 $pathtobin -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask \
204 >> $tmpf 2>&1 "${cmdline[@]}";;
206 $pathtobin -np $((ntask*nclnt)) >> $tmpf 2>&1 "${cmdline[@]}";;
210 if [ $runior = "mpirun" ]; then
211 mpi_client_file ${clients[@]}
214 # convert params to actual numbers
215 min_per_client_size=`parse_number $min_per_client_size`
216 min_total_size=`parse_number $min_total_size`
218 rsize_lo=`parse_number $rsize_lo`
219 rsize_hi=`parse_number $rsize_hi`
221 # check on actual numbers of client nodes
222 nclients=`countnodes ${clients[@]}`
223 if ((clients_hi > nclients)); then clients_hi=$nclients; fi
225 for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
226 pp_rsize=`pp_number $rsize`
228 for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
229 test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
231 per_client_size=$((min_total_size/nclnt))
232 if ((per_client_size < min_per_client_size)); then
233 per_client_size=$min_per_client_size
236 for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
237 per_task_size=$((per_client_size/ntask))
238 if ((per_task_size%rsize != 0)); then
239 per_task_size=$(((per_task_size/rsize + 1)*rsize))
241 total_size=`pp_number $((per_task_size*nclnt*ntask))`
243 hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
244 $total_size $pp_rsize $nclnt $ntask`
245 print_summary -n "$hdrstr"
247 for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
248 test=${tests[$test_idx]}
250 print_summary -n "$test "
251 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
255 if [ "$test" = "remount" ]; then
256 echo "=> $remount" >> $tmpf
257 if [ "$runior" = "pdsh" ]; then
258 $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
260 $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
267 echo "Completion Status: $status" >> $tmpf
275 # check lustre is mounted everywhere it's needed
276 cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
277 echo "=> Mount Check: $cmd" >> $tmpf
278 if [ "$runior" = "pdsh" ]; then
279 $pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
283 echo "Completion Status: $status" >> $tmpf
287 print_summary "Lustre NOT mounted on $lustre somewhere"
293 -o${testfile} # test file prefix
294 -b${per_task_size} # bytes per task
295 -t${rsize} # record size
296 -e # fsync before close
302 # keep the test file(s) unless this is the last test
303 ((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
305 # use the existing test file(s) unless this is the first test
306 ((test_idx > 0)) && cmdline[$((idx++))]="-E"
309 ((file_per_task)) && cmdline[$((idx++))]="-F"
312 *write*) cmdline[$((idx++))]="-w"
314 *) cmdline[$((idx++))]="-r"
318 echo "=> ${cmdline[@]}" >> $tmpf
323 echo "Completion Status: $status" >> $tmpf
328 result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
329 END {if (!found) print \"ERROR\"}"`
336 str=`printf "%8s" "$result"`
337 print_summary -n "$str "