Whamcloud - gitweb
a2d67245600ce5563d46d43488c4c9b2ac5422d8
[fs/lustre-release.git] / lustre-iokit / ior-survey / ior-survey
1 #!/bin/bash
2
3 # the value of these can be set in the environment.
4 # This test assumes a typical pdsh naming scheme, where
5 # node names can be expressed as a string
6 # forllowed by a number 
7 # cluster name (all node names are this followed by the node number)
8 cluster=${cluster:-""}
9 if [ -z "$cluster" ];then  echo  "cluster not defined"; exit 1; fi
10
11 # client node numbers (individual numbers or inclusive ranges)
12 clients=${clients:-""}
13 if [ -z "$clients" ]; then echo "clients not defined"; exit 1; fi
14
15 # numbers of clients to survey
16 clients_lo=${clients_lo:-1}
17 clients_hi=${clients_hi:-3}
18 clients_iterator=${clients_iterator:-"+=1"}
19
20 # numbers of tasks per client to survey
21 tasks_per_client_lo=${task_per_client_lo:-1}
22 tasks_per_client_hi=${tasks_per_client_hi:-8}
23 tasks_per_client_iterator=${tasks_per_client_iterator:-"*=2"}
24
25 # record sizes to survey
26 rsize_lo=${rsize_lo:-1M}
27 rsize_hi=${rsize_hi:-1M}
28 rsize_iterator=${rsize_iterator:-"*=2"}
29
30 ## which tests to run (first must be write)
31 # clear_cache)   not really a test; just uncache everything
32 # *write*)   write
33 # *)         read
34 tests=(write rewrite clear_cache read reread)
35
36 # total # bytes written/read by any client node
37 min_per_client_size=${min_per_client_size:-"1G"}
38 min_total_size=${min_total_size:-"2G"}
39
40 # should each task do I/O to its own file?
41 file_per_task=${file_per_task:-1}
42
43 # the binaries
44 IOR=${IOR:-"/usr/local/sbin/IOR"}
45 llmount=${llmount:-"llmount"}
46 # Select mpirun, yod, or pdsh
47 fanout_cmd=${fanout_cmd:-"pdsh"}
48 # mpirun still uses pdsh for cleanup
49 pdsh=${pdsh:-"pdsh"}
50 pdsh_args="-R ssh -S -b -w "
51
52 # the result file prefix (date/time + hostname makes unique)
53 rslt_loc=${rslt_loc:-"/tmp"}
54 rslt=${rslt:-"$rslt_loc/ior_survey_`date +%F@%R`_`uname -n`"}
55
56 # where lustre is mounted on the clients
57 lustre=${lustre:-"/mnt/lustre"}
58
59 # basename of the test file(s)
60 testfile=${testfile:-"${lustre}/ior/ior_survey_testfile"}
61
62 #don't spin for MPI completions
63 export LIBELAN_WAITTYPE=0
64
65 ################################################################################
66 # dont change stuff below here unless you know what you're doing...
67
68 # This is to allow use of yod, pdsh, etc.
69 fanout() {
70         local clients=$1; shift
71         local tmpfile=$1; shift
72         local result
73         case $fanout_cmd in
74         'pdsh')
75                 $fanout_cmd $pdsh_args "$clients" "$@" >> $tmpfile 2>&1
76                 echo $?
77                 return
78                 ;;
79         'mpirun')
80                 # horrible misuse of globals
81                 $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
82                 echo $?
83                 return
84                 ;;
85         'yod')
86                 # and another
87                 $fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
88                 echo $?
89                 return
90                 ;;
91                 
92         *)
93                 echo "255"
94                 return
95                 ;;
96         esac
97 }
98
99 dump_cache() {
100         # we are assuming mpi uses will also have pdsh
101         local clients=$1;shift
102         local tmpfile=$1;shift
103         clear_cache='for LRU in /proc/fs/lustre/ldlm/namespaces/*/lru_size; 
104                 do sudo /bin/bash -c "echo clear > $LRU"; done'
105         echo "=> $clear_cache" >> $tmpfile
106         $pdsh $pdsh_args "$test_clients" "$clear_cache" >> $tmpfile 2>&1 
107         status=$?
108         echo "Completion Status: $status" >> $tmpfile
109
110         if ((status)); then
111               echo "ERROR"
112         else
113               echo "OK"
114         fi
115 }
116 count_range() {
117     echo $1 | awk '{ nvals=split($1, vals, "-");\
118                      if (nvals == 1) print 1;\
119                      else if (nvals == 2) printf "%d\n", vals[2] - vals[1] + 1;}'
120 }
121
122 base_range() {
123     echo $1 | awk '{ split($1, vals, "-"); print vals[1]; }'
124 }
125
126 idx2nodenum() {
127     local n=$1; shift
128     while ((1)); do
129         local range=$1; shift
130         if [ -z "$range" ]; then
131             return
132         fi
133         chunk=`count_range $range`
134         if ((chunk > $n)); then
135             base=`base_range $range`
136             echo $((base + n))
137             return
138         fi
139         n=$((n-chunk))
140     done
141 }
142
143 n2noderange() {
144     local n=$1; shift
145     sep=""
146     nodes="["
147     while ((n > 0)); do
148         local range=$1; shift
149         if [ -z "$range" ]; then
150             return
151         fi
152         local base=`base_range $range`
153         local chunk=`count_range $range`
154         if ((chunk > $n)); then chunk=$n; fi
155         local nodes="${nodes}${sep}${base}"; sep=","
156         if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
157         n=$((n-chunk))
158     done
159     echo "${nodes}]"
160 }
161
162 countnodes() {
163     local radix=16384
164     local n=0
165     while ((radix > 0)); do
166         local nodes=`n2noderange $((n+radix)) $@`
167         if [ -n "$nodes" ]; then
168             n=$((n+radix))
169         fi
170         radix=$((radix/2))
171     done
172     echo $n
173 }
174
175 parse_number() {
176     local str=$1
177     case $str in
178         *G|*g) n=`echo $str | sed 's/[gG]//'`; echo $((n*1024*1024*1024));;
179         *M|*m) n=`echo $str | sed 's/[Mm]//'`; echo $((n*1024*1024));;
180         *K|*k) n=`echo $str | sed 's/[Kk]//'`; echo $((n*1024));;
181         *)     echo $1;;
182     esac
183 }
184
185 pp_number() {
186     local n=$1
187     local G=$((1024*1024*1024))
188     local M=$((1024*1024))
189     local K=$((1024))
190     if ((n%G == 0 && n >= $G)); then
191         echo "$((n/G))G"
192     elif ((n%M == 0 && n >= $M)); then
193         echo "$((n/M))M"
194     elif ((n%K == 0 && n >= $K)); then
195         echo "$((n/K))K"
196     else
197         echo $n
198     fi
199 }
200
201 if [ ${#tests[@]} -eq 0 -o "${tests[0]}" != "write" ]; then
202     echo "First test must be 'write'" 1>&2
203     exit 1
204 fi
205
206 rsltf="${rslt}.summary"
207 workf="${rslt}.detail"
208 echo -n > $rsltf
209 echo -n > $workf
210
211 print_summary () {
212     if [ "$1" = "-n" ]; then
213         minusn=$1; shift
214     else
215         minusn=""
216     fi
217     echo $minusn "$*" >> $rsltf
218     echo $minusn "$*"
219 }
220
221 check_mount() {
222     local lustre=$1; shift
223     local tmpb=$1; shift
224     local clients=$1; shift
225     local tmpfile=${tmpb}_tmp
226     # check lustre is mounted everywhere it's needed
227    cmd="grep $lustre /proc/mounts"
228    $pdsh $pdsh_args "$clients" "$cmd" >> $tmpfile
229    status=$?
230         if (($status)); then
231                 print_summary "Lustre NOT mounted on $lustre somewhere"
232         exit 1
233         fi
234
235 }
236 # convert params to actual numbers
237 min_per_client_size=`parse_number $min_per_client_size`
238 min_total_size=`parse_number $min_total_size`
239
240 rsize_lo=`parse_number $rsize_lo`
241 rsize_hi=`parse_number $rsize_hi`
242
243 # check on actual numbers of client nodes
244 nclients=`countnodes ${clients[@]}`
245 if ((clients_hi > $nclients)); then clients_hi=$nclients; fi
246
247 cur_date=`date`
248 machine=`uname -n`
249 script_name=`echo $0 | cut -d "/" -f2`
250
251 echo "$cur_date $script_name on $lustre from $machine" >> $workf
252
253 for ((rsize=$rsize_lo; rsize<=$rsize_hi; rsize$rsize_iterator)); do
254     pp_rsize=`pp_number $rsize`
255
256     for ((nclnt=$clients_lo; nclnt<=$clients_hi; nclnt$clients_iterator)); do
257         test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
258         echo $test_clients
259         if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ];then
260                 check_mount $lustre $workf $test_clients
261         fi
262         per_client_size=$((min_total_size/nclnt))
263         if ((per_client_size < $min_per_client_size)); then
264             per_client_size=$min_per_client_size
265         fi
266
267         for ((ntask=$tasks_per_client_lo; ntask <= $tasks_per_client_hi; \
268                 ntask$tasks_per_client_iterator)); do
269             per_task_size=$((per_client_size/ntask))
270             if ((per_task_size%rsize != 0)); then
271                 per_task_size=$(((per_task_size/rsize + 1)*rsize))
272             fi
273             total_size=`pp_number $((per_task_size*nclnt*ntask))`
274             
275             hdrstr=`printf "Total: %5sB rsize: %4sB clients: %4d tasks: %3d: " \
276                 $total_size $pp_rsize $nclnt $ntask`
277             print_summary -n "$hdrstr"
278
279             for ((test_idx=0; test_idx < ${#tests[@]}; test_idx++)); do
280                 test=${tests[$test_idx]}
281                 
282                 print_summary -n "$test "
283                 echo "===========> ${hdrstr} on $test_clients doing $test" >> $workf
284                 tmpf=${workf}_tmp
285                 echo -n > $tmpf
286
287                 if [ "$test" = "clear_cache" ]; then
288                     if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ]; then
289                                 result=`dump_cache $test_clients $tmpf`
290                      else
291                                 echo "Haven't figured out how to clear cache" >> $tmpf
292                                 result="N/A"
293                      fi
294                 else
295
296                     cmdline=(
297                     $IOR                     # the command
298                     -o${testfile}            # test file prefix
299                     -b${per_task_size}       # bytes per task
300                     -t${rsize}               # record size
301                     -e                       # fsync before close
302                     -q                       # quit on error
303                     )
304
305                     idx=${#cmdline[@]}
306
307                     # keep the test file(s) unless this is the last test
308                     #((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
309                     cmdline[$((idx++))]="-k"
310
311                     # use the existing test file(s) unless this is the first test
312                     ((test_idx > 0)) && cmdline[$((idx++))]="-E"
313
314                     # file-per-task
315                     (($file_per_task)) && cmdline[$((idx++))]="-F"
316
317                     case "$test" in
318                     *write*) cmdline[$((idx++))]="-w"
319                              awkstr="Max Write";;
320                     *)       cmdline[$((idx++))]="-r"
321                              awkstr="Max Read";;
322                     esac
323
324                     echo "=> ${cmdline[@]}" >> $tmpf
325         
326                     status=`fanout $test_clients $tmpf ${cmdline[@]}`
327
328                     echo "Completion Status: $status" >> $tmpf
329                
330                     if (($status)); then
331                         result="ERROR"
332                     else
333                         # pdsh adds an extra field
334                         if [ "$fanout_cmd" = "pdsh" ]; then 
335                                 result=`awk < $tmpf "/$awkstr/ {print $ 4; found=1; exit}\
336                                              END       {if (!found) print \"ERROR\"}"`
337                         else
338                                 result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
339                                              END       {if (!found) print \"ERROR\"}"`
340                         fi
341                     fi
342                 fi
343
344                 cat $tmpf >> $workf
345                 rm $tmpf
346
347                 str=`printf "%8s" "$result"`
348                 print_summary -n "$str "
349             done
350             print_summary ""
351         done
352     done
353 done
354