3 # the value of these can be set in the environment.
4 # This test assumes a typical pdsh naming scheme, where
5 # node names can be expressed as a string
6 # forllowed by a number
7 # cluster name (all node names are this followed by the node number)
9 if [ -z "$cluster" ];then echo "cluster not defined"; exit 1; fi
11 # client node numbers (individual numbers or inclusive ranges)
12 clients=${clients:-""}
13 if [ -z "$clients" ]; then echo "clients not defined"; exit 1; fi
15 # numbers of clients to survey
16 clients_lo=${clients_lo:-1}
17 clients_hi=${clients_hi:-3}
18 clients_iterator=${clients_iterator:-"+=1"}
20 # numbers of tasks per client to survey
21 tasks_per_client_lo=${task_per_client_lo:-1}
22 tasks_per_client_hi=${tasks_per_client_hi:-8}
23 tasks_per_client_iterator=${tasks_per_client_iterator:-"*=2"}
25 # record sizes to survey
26 rsize_lo=${rsize_lo:-1M}
27 rsize_hi=${rsize_hi:-1M}
28 rsize_iterator=${rsize_iterator:-"*=2"}
30 ## which tests to run (first must be write)
31 # clear_cache) not really a test; just uncache everything
34 tests=(write rewrite clear_cache read reread)
36 # total # bytes written/read by any client node
37 min_per_client_size=${min_per_client_size:-"1G"}
38 min_total_size=${min_total_size:-"2G"}
40 # should each task do I/O to its own file?
41 file_per_task=${file_per_task:-1}
44 IOR=${IOR:-"/usr/local/sbin/IOR"}
45 llmount=${llmount:-"llmount"}
46 # Select mpirun, yod, or pdsh
47 fanout_cmd=${fanout_cmd:-"pdsh"}
48 # mpirun still uses pdsh for cleanup
50 pdsh_args="-R ssh -S -b -w "
52 # the result file prefix (date/time + hostname makes unique)
53 rslt_loc=${rslt_loc:-"/tmp"}
54 rslt=${rslt:-"$rslt_loc/ior_survey_`date +%F@%R`_`uname -n`"}
56 # where lustre is mounted on the clients
57 lustre=${lustre:-"/mnt/lustre"}
59 # basename of the test file(s)
60 testfile=${testfile:-"${lustre}/ior/ior_survey_testfile"}
62 #don't spin for MPI completions
63 export LIBELAN_WAITTYPE=0
65 ################################################################################
66 # dont change stuff below here unless you know what you're doing...
68 # This is to allow use of yod, pdsh, etc.
70 local clients=$1; shift
71 local tmpfile=$1; shift
75 $fanout_cmd $pdsh_args "$clients" "$@" >> $tmpfile 2>&1
80 # horrible misuse of globals
81 $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
87 $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
100 # we are assuming mpi uses will also have pdsh
101 local clients=$1;shift
102 local tmpfile=$1;shift
103 clear_cache='lctl set_param ldlm.namespaces.*.lru_size=clear'
104 echo "=> $clear_cache" >> $tmpfile
105 $pdsh $pdsh_args "$test_clients" "$clear_cache" >> $tmpfile 2>&1
107 echo "Completion Status: $status" >> $tmpfile
116 echo $1 | awk '{ nvals=split($1, vals, "-");\
117 if (nvals == 1) print 1;\
118 else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
122 echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
128 local range=$1; shift
129 if [ -z "$range" ]; then
132 chunk=`count_range $range`
133 if ((chunk > $n)); then
134 base=`base_range $range`
147 local range=$1; shift
148 if [ -z "$range" ]; then
151 local base=`base_range $range`
152 local chunk=`count_range $range`
153 if ((chunk > $n)); then chunk=$n; fi
154 local nodes="${nodes}${sep}${base}"; sep=","
155 if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
164 while ((radix > 0)); do
165 local nodes=`n2noderange $((n+radix)) $@`
166 if [ -n "$nodes" ]; then
177 *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
178 *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
179 *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
186 local G=$((1024*1024*1024))
187 local M=$((1024*1024))
189 if ((n%G == 0 && n >= $G)); then
191 elif ((n%M == 0 && n >= $M)); then
193 elif ((n%K == 0 && n >= $K)); then
200 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
201 echo "First test must be 'write'" 1>&2
205 rsltf="${rslt}.summary"
206 workf="${rslt}.detail"
211 if [ "$1" = "-n" ]; then
216 echo $minusn "$*" >> $rsltf
221 local lustre=$1; shift
223 local clients=$1; shift
224 local tmpfile=${tmpb}_tmp
225 # check lustre is mounted everywhere it's needed
226 cmd="grep $lustre /proc/mounts"
227 $pdsh $pdsh_args "$clients" "$cmd" >> $tmpfile
230 print_summary "Lustre NOT mounted on $lustre somewhere"
235 # convert params to actual numbers
236 min_per_client_size=`parse_number $min_per_client_size`
237 min_total_size=`parse_number $min_total_size`
239 rsize_lo=`parse_number $rsize_lo`
240 rsize_hi=`parse_number $rsize_hi`
242 # check on actual numbers of client nodes
243 nclients=`countnodes ${clients[@]}`
244 if ((clients_hi > $nclients)); then clients_hi=$nclients; fi
248 script_name=`echo $0 | cut -d "/" -f2`
250 echo "$cur_date $script_name on $lustre from $machine" >> $workf
252 for ((rsize=$rsize_lo; rsize<=$rsize_hi; rsize$rsize_iterator)); do
253 pp_rsize=`pp_number $rsize`
255 for ((nclnt=$clients_lo; nclnt<=$clients_hi; nclnt$clients_iterator)); do
256 test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
258 if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ];then
259 check_mount $lustre $workf $test_clients
261 per_client_size=$((min_total_size/nclnt))
262 if ((per_client_size < $min_per_client_size)); then
263 per_client_size=$min_per_client_size
266 for ((ntask=$tasks_per_client_lo; ntask <= $tasks_per_client_hi; \
267 ntask$tasks_per_client_iterator)); do
268 per_task_size=$((per_client_size/ntask))
269 if ((per_task_size%rsize != 0)); then
270 per_task_size=$(((per_task_size/rsize + 1)*rsize))
272 total_size=`pp_number $((per_task_size*nclnt*ntask))`
274 hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
275 $total_size $pp_rsize $nclnt $ntask`
276 print_summary -n "$hdrstr"
278 for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
279 test=${tests[$test_idx]}
281 print_summary -n "$test "
282 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
286 if [ "$test" = "clear_cache" ]; then
287 if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ]; then
288 result=`dump_cache $test_clients $tmpf`
290 echo "Haven't figured out how to clear cache" >> $tmpf
297 -o${testfile} # test file prefix
298 -b${per_task_size} # bytes per task
299 -t${rsize} # record size
300 -e # fsync before close
306 # keep the test file(s) unless this is the last test
307 #((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
308 cmdline[$((idx++))]="-k"
310 # use the existing test file(s) unless this is the first test
311 ((test_idx > 0)) && cmdline[$((idx++))]="-E"
314 (($file_per_task)) && cmdline[$((idx++))]="-F"
317 *write*) cmdline[$((idx++))]="-w"
319 *) cmdline[$((idx++))]="-r"
323 echo "=> ${cmdline[@]}" >> $tmpf
325 status=`fanout $test_clients $tmpf ${cmdline[@]}`
327 echo "Completion Status: $status" >> $tmpf
332 # pdsh adds an extra field
333 if [ "$fanout_cmd" = "pdsh" ]; then
334 result=`awk < $tmpf "/$awkstr/ {print $ 4; found=1; exit}\
335 END {if (!found) print \"ERROR\"}"`
337 result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
338 END {if (!found) print \"ERROR\"}"`
346 str=`printf "%8s" "$result"`
347 print_summary -n "$str "